From 5751efa8cbeb021a4a690ca6adee43e4bd7232d8 Mon Sep 17 00:00:00 2001 From: Camilla Pacifici Date: Fri, 21 Jul 2023 11:20:47 -0400 Subject: [PATCH 01/62] Initial commit of NIRISS advanced notebooks --- .../00_mast_query_progID.ipynb | 160 +++ .../01_check_files_headers_progID.ipynb | 128 +++ .../02_run_pipeline_wcs_flatfield.ipynb | 164 +++ .../03_imviz_level2.ipynb | 251 +++++ .../04_run_pipeline_imaging_level2and3.ipynb | 537 ++++++++++ .../05_catalog_for_spectral_extraction.ipynb | 102 ++ .../06_run_pipeline_spec2.ipynb | 253 +++++ .../07_specviz2d_extract1d.ipynb | 171 ++++ .../08_compare_extractions.ipynb | 962 ++++++++++++++++++ 9 files changed, 2728 insertions(+) create mode 100644 notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb create mode 100644 notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb create mode 100644 notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb create mode 100644 notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb create mode 100644 notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb create mode 100644 notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb create mode 100644 notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb create mode 100644 notebooks/NIRISS_WFSS_advanced/07_specviz2d_extract1d.ipynb create mode 100755 notebooks/NIRISS_WFSS_advanced/08_compare_extractions.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb new file mode 100644 index 000000000..dd2bca8c3 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb @@ -0,0 +1,160 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "592da8da", + "metadata": {}, + "source": [ + "## Get observations from program ID\n", + "Inspired by the [JWebbinar session about MAST](https://github.com/spacetelescope/jwebbinar_prep/blob/main/mast_session/Crowded_Field/Crowded_Field.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e92dee7", + "metadata": {}, + "outputs": [], + "source": [ + "from astropy import units as u\n", + "from astropy.coordinates import SkyCoord\n", + "from astropy.io import fits\n", + "from astroquery.mast import Observations\n", + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "id": "9447551a", + "metadata": {}, + "source": [ + "Select the proposal ID, instrument, and some useful keywords (filters in this case)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "569efa6d", + "metadata": {}, + "outputs": [], + "source": [ + "obs_table = Observations.query_criteria(obs_collection=[\"JWST\"], \n", + " instrument_name=[\"NIRISS/IMAGE\", \"NIRISS/WFSS\"],\n", + " provenance_name=[\"CALJWST\"], # Executed observations\n", + " filters=['F115W','F150W','F200W'],\n", + " proposal_id=[2079],\n", + " )\n", + "obs_table" + ] + }, + { + "cell_type": "markdown", + "id": "9daaeaf9", + "metadata": {}, + "source": [ + "I could list all the products, but if there are too many it could time out. Let us devide the observations in baches and then filter and download one batch at a time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8a653a9", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 5\n", + "# Let's split up ``obs_table`` into batches according to our batch size.\n", + "obs_batches = [obs_table[i:i+batch_size] for i in range(0, len(obs_table), batch_size)]\n", + "single_group = obs_batches[0] # Useful to inspect the files obtained from one group\n", + "print(\"How many batches?\", len(obs_batches))" + ] + }, + { + "cell_type": "markdown", + "id": "10c7d144", + "metadata": {}, + "source": [ + "Select the type of products needed. I do one at a time. I need:\n", + "- rate images\n", + " - productType=[\"SCIENCE\"]\n", + " - productSubGroupDescription=['RATE']\n", + " - calib_level=[2]\n", + "- level 2 associations for both spectroscopy and imaging\n", + " - productType=[\"INFO\"]\n", + " - productSubGroupDescription=['ASN']\n", + " - calib_level=[2]\n", + "- level 3 associations for imaging\n", + " - productType=[\"INFO\"]\n", + " - productSubGroupDescription=['ASN']\n", + " - dataproduct_type=[\"image\"]\n", + " - calib_level=[3]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "616ff15c", + "metadata": {}, + "outputs": [], + "source": [ + "# Now let's get the products for each batch of observations, and filter down to only the products of interest.\n", + "for index, batch in enumerate(obs_batches):\n", + "#for index, batch in enumerate(single_group): # Useful to inspect the files obtained from one batch\n", + " \n", + " # Progress indicator...\n", + " print('\\n'+f'Batch #{index+1}')\n", + " \n", + " # Make a list of the `obsid` identifiers from our Astropy table of observations to get products for.\n", + " obsids = batch['obsid']\n", + " print('Working with the following ``obsid``s:')\n", + " print(list(obsids))\n", + " \n", + " # Get list of products \n", + " products = Observations.get_product_list(obsids)\n", + " \n", + " # Filter the products to only get science products of calibration level 3 \n", + " filtered_products = Observations.filter_products(products, \n", + " productType=[\"INFO\"],\n", + " productSubGroupDescription=['ASN'],\n", + " dataproduct_type=[\"image\"],\n", + " calib_level=[3]\n", + " )\n", + " #filtered_products\n", + " # Download products for these records.\n", + " print('Products downloaded:')\n", + " print(filtered_products['productFilename'])\n", + " #print(filtered_products)\n", + " manifest = Observations.download_products(filtered_products)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de034fed", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb new file mode 100644 index 000000000..6a1ec04c3 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "continental-machinery", + "metadata": {}, + "source": [ + "# Create list of files from MAST download" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "nasty-belize", + "metadata": {}, + "outputs": [], + "source": [ + "from astropy.io import fits\n", + "import pandas as pd\n", + "import glob" + ] + }, + { + "cell_type": "markdown", + "id": "95d38a86", + "metadata": {}, + "source": [ + "## Get list of files with header keywords" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3db7c79f", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "df = pd.DataFrame(columns=[\"FILENAME\", \"TARG_RA\", \"TARG_DEC\", \"FILTER\", \"PUPIL\", \"PATT_NUM\", \"NUMDTHPT\", \"XOFFSET\", \"YOFFSET\"])\n", + "\n", + "for file in glob.glob(\"./mastDownload/JWST/jw02079004*/*rate.fits\"):\n", + " image = fits.open(file)\n", + " \n", + " df2 = pd.DataFrame({\"FILENAME\" : [image[0].header[\"FILENAME\"]],\n", + " \"TARG_RA\" : [image[0].header[\"TARG_RA\"]],\n", + " \"TARG_DEC\" : [image[0].header[\"TARG_DEC\"]],\n", + " \"FILTER\" : [image[0].header[\"FILTER\"]],\n", + " \"PUPIL\" : [image[0].header[\"PUPIL\"]],\n", + " \"PATT_NUM\" : [image[0].header[\"PATT_NUM\"]],\n", + " \"NUMDTHPT\" : [image[0].header[\"NUMDTHPT\"]],\n", + " \"XOFFSET\" : [image[0].header[\"XOFFSET\"]],\n", + " \"YOFFSET\" : [image[0].header[\"YOFFSET\"]]\n", + " })\n", + " \n", + " df = pd.concat([df, df2], ignore_index = True, axis = 0)\n", + " \n", + "dfsort = df.sort_values('FILENAME', ignore_index = True)" + ] + }, + { + "cell_type": "markdown", + "id": "7002e5c9", + "metadata": {}, + "source": [ + "## Save to csv file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a77d81da", + "metadata": {}, + "outputs": [], + "source": [ + "dfsort.to_csv(\"./list_ngdeep_rate.csv\",sep=',')" + ] + }, + { + "cell_type": "markdown", + "id": "719e5a24", + "metadata": {}, + "source": [ + "## Move files to preferred location" + ] + }, + { + "cell_type": "markdown", + "id": "12a195ca", + "metadata": {}, + "source": [ + "At this point, I move the rate files and the association files to a preferred place on my machine:\n", + "- all rate files under NGDEEP/rate\n", + "- level 2 association files under NGDEEP/asn_level2\n", + "- level 3 association files under NGDEEP/asn_level3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bc1b5c4", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb b/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb new file mode 100644 index 000000000..9cbee229c --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb @@ -0,0 +1,164 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1cea74b7", + "metadata": {}, + "source": [ + "# Run assign_wcs and flatfielding" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ee53c65", + "metadata": {}, + "outputs": [], + "source": [ + "# Update the CRDS path to your local directory\n", + "%env CRDS_PATH=/Users/cpacifici/crds_cache_july2023\n", + "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu\n", + " \n", + "import os\n", + "import glob\n", + "#jwst pipeline related modules\n", + "import jwst\n", + "from jwst import datamodels\n", + "# Individual steps that make up calwebb_image2\n", + "from jwst.background import BackgroundStep\n", + "from jwst.assign_wcs import AssignWcsStep\n", + "from jwst.flatfield import FlatFieldStep\n", + "from jwst.photom import PhotomStep\n", + "from jwst.resample import ResampleStep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df3bb687", + "metadata": {}, + "outputs": [], + "source": [ + "print('jwst:', jwst.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4084d5a5", + "metadata": {}, + "outputs": [], + "source": [ + "# Update appropriately (got the rate files from MAST)\n", + "data_dir_in = \"/Users/cpacifici/DATA/NGDEEP/rate/*\"\n", + "data_dir_out = \"/Users/cpacifici/DATA/NGDEEP/wcs_flat/\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97183293", + "metadata": {}, + "outputs": [], + "source": [ + "# List the rate files\n", + "ratefiles = glob.glob(data_dir_in+'*')" + ] + }, + { + "cell_type": "raw", + "id": "66bc6d0a", + "metadata": {}, + "source": [ + "# Check the name, pupil, and filter of all files (this step is not necessary, just a sanity check)\n", + "for file in ratefiles:\n", + " model = datamodels.open(file)\n", + " print(file)\n", + " print(model.meta.instrument.name)\n", + " print(model.meta.instrument.pupil)\n", + " print(model.meta.instrument.filter)\n", + " print()\n", + " model.close()" + ] + }, + { + "cell_type": "markdown", + "id": "2e145cf6", + "metadata": {}, + "source": [ + "**Developer note:** it would be better to update these to use the `call()` method instead of the `run()` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b143bddf", + "metadata": {}, + "outputs": [], + "source": [ + "# Run assign_wcs\n", + "awcsfiles = []\n", + "for file in ratefiles:\n", + " model = datamodels.open(file)\n", + " name = file.split('/')[-1]\n", + " newname = name.replace('rate.fits', 'assignwcsstep.fits')\n", + " assign_wcs_step = AssignWcsStep()\n", + " assign_wcs_step.output_dir = data_dir_out\n", + " assign_wcs_step.save_results = True\n", + " #run the step\n", + " assign_wcs_step.run(file)\n", + " awcsfiles.append(data_dir_out+newname)\n", + " print(newname)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6e6f72a", + "metadata": {}, + "outputs": [], + "source": [ + "# Run flatfielding\n", + "fffiles = []\n", + "for file in awcsfiles:\n", + " newname = file.replace('assignwcsstep.fits', 'flatfieldstep.fits')\n", + " flatfield_step = FlatFieldStep()\n", + " flatfield_step.output_dir = data_dir_out\n", + " flatfield_step.save_results = True \n", + " #run the step\n", + " flatfield_step.run(file)\n", + " fffiles.append(newname)\n", + " print(newname)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f46c65fa", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb b/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb new file mode 100644 index 000000000..d07713bc0 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb @@ -0,0 +1,251 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "28608b44", + "metadata": {}, + "source": [ + "# Check flat fielded images from NGDEEP" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "216f4a35", + "metadata": {}, + "outputs": [], + "source": [ + "from jdaviz import Imviz\n", + "import warnings\n", + "from astropy.io import ascii\n", + "from astropy.table import Table\n", + "from astropy.coordinates import SkyCoord, Angle\n", + "import astropy.units as u\n", + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "id": "d4c0f2ce", + "metadata": {}, + "source": [ + "## Read list of files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf65d59c", + "metadata": {}, + "outputs": [], + "source": [ + "listrate_file = './list_ngdeep_rate.csv'\n", + "listrate = Table.read(listrate_file)\n", + "listrate[46::]" + ] + }, + { + "cell_type": "markdown", + "id": "3a7a85e7", + "metadata": {}, + "source": [ + "## Check astrometry on direct images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "352872bc", + "metadata": {}, + "outputs": [], + "source": [ + "filedir = '/Users/cpacifici/DATA/NGDEEP/wcs_flat/'\n", + "clearim = []\n", + "datalabel = []\n", + "for ii, file in enumerate(listrate['FILENAME']):\n", + " if listrate['FILTER'][ii]=='CLEAR':\n", + " name = file.split('/')[-1]\n", + " newname = name.replace('rate.fits', 'flatfieldstep.fits')\n", + " clearim.append(filedir+newname)\n", + " datalabel.append(listrate['PUPIL'][ii])\n", + "clearim" + ] + }, + { + "cell_type": "markdown", + "id": "9144f386", + "metadata": {}, + "source": [ + "### Use Imviz batch load since it is many images\n", + "It takes about 1 minute to load all the images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efbedea9", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "imviz = Imviz()\n", + "for ii in range(len(clearim)):\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter('ignore')\n", + " imviz.load_data(clearim[ii], data_label=datalabel[ii])\n", + "imviz.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6ebfa64", + "metadata": {}, + "outputs": [], + "source": [ + "viewer = imviz.default_viewer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ed6c0c4", + "metadata": {}, + "outputs": [], + "source": [ + "linking = imviz.plugins['Links Control']\n", + "linking.link_type = 'WCS'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53b3db0f", + "metadata": {}, + "outputs": [], + "source": [ + "# This also takes a bit of time (~30 seconds) to loop through all the images\n", + "plotopt = imviz.plugins['Plot Options']\n", + "plotopt.multiselect = True\n", + "plotopt.select_all(viewers=True, layers=True)\n", + "plotopt.stretch_preset = '99.5%'\n", + "plotopt.multiselect = False\n", + "#plotopt.stretch_function = 'Arcsinh'\n", + "#plotopt.stretch_vmin = -0.002\n", + "#plotopt.stretch_vmax = 0.16" + ] + }, + { + "cell_type": "markdown", + "id": "5a2a5552", + "metadata": {}, + "source": [ + "### Blink\n", + "Using the \"b\" key when on the viewer." + ] + }, + { + "cell_type": "markdown", + "id": "54cebc13", + "metadata": {}, + "source": [ + "**Data note**: there is a problem with the astrometry. The F200W filter does not match the others. I am not sure if F200W is the problem or the other two are the problem. Fixing the astrometry is out of scope for this notebook." + ] + }, + { + "cell_type": "markdown", + "id": "015c5f3c", + "metadata": {}, + "source": [ + "## Check direct images and their corresponding WFSS images\n", + "Selecting by hand from the table to use direct and dispersed images on the same dither position." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0c1f449", + "metadata": {}, + "outputs": [], + "source": [ + "dir_f115w = filedir + 'jw02079004001_04101_00004_nis_flatfieldstep.fits'\n", + "disp_f115w = filedir + 'jw02079004001_05101_00001_nis_flatfieldstep.fits'\n", + "dir_f150w = filedir + 'jw02079004002_10101_00004_nis_flatfieldstep.fits'\n", + "disp_f150w = filedir + 'jw02079004002_11101_00001_nis_flatfieldstep.fits'\n", + "dir_f200w = filedir + 'jw02079004003_04101_00004_nis_flatfieldstep.fits'\n", + "disp_f200w = filedir + 'jw02079004003_05101_00001_nis_flatfieldstep.fits'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da61c2a4", + "metadata": {}, + "outputs": [], + "source": [ + "imviz2 = Imviz()\n", + "with warnings.catch_warnings():\n", + " warnings.simplefilter('ignore')\n", + " imviz2.load_data(dir_f115w, data_label='F115W dir')\n", + " imviz2.load_data(disp_f115w, data_label='F115W wfss')\n", + " imviz2.load_data(dir_f150w, data_label='F150W dir')\n", + " imviz2.load_data(disp_f150w, data_label='F150W wfss')\n", + " imviz2.load_data(dir_f200w, data_label='F200W dir')\n", + " imviz2.load_data(disp_f200w, data_label='F200W wfss')\n", + "imviz2.show()" + ] + }, + { + "cell_type": "markdown", + "id": "037d515a", + "metadata": {}, + "source": [ + "Split direct images and dispersed images in two separate viewers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c8892f4", + "metadata": {}, + "outputs": [], + "source": [ + "plotopt = imviz2.plugins['Plot Options']\n", + "plotopt.multiselect = True\n", + "plotopt.select_all(viewers=True, layers=True)\n", + "plotopt.stretch_preset = '99.5%'\n", + "plotopt.multiselect = False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba0d07ff", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb new file mode 100644 index 000000000..ecfc7fe18 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb @@ -0,0 +1,537 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7a935d7c", + "metadata": {}, + "source": [ + "# Run pipeline and create catalog of sources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "028352f4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"CRDS_PATH\"] = \"/Users/cpacifici/crds_cache_july2023/\" # set appropriate path\n", + "os.environ[\"CRDS_SERVER_URL\"] = \"https://jwst-crds.stsci.edu\"\n", + "\n", + "import glob\n", + "import shutil\n", + "\n", + "import jwst\n", + "from jwst.pipeline import Image2Pipeline\n", + "from jwst.pipeline import Image3Pipeline\n", + "\n", + "from astropy.table import Table\n", + "\n", + "import numpy as np\n", + "\n", + "import json\n", + "\n", + "from jdaviz import Imviz\n", + "import warnings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae10a560", + "metadata": {}, + "outputs": [], + "source": [ + "print('jwst:', jwst.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60437b9c", + "metadata": {}, + "outputs": [], + "source": [ + "# Update appropriately\n", + "data_dir_in = \"/Users/cpacifici/DATA/NGDEEP/rate/\"\n", + "data_dir_asn_level2 = \"/Users/cpacifici/DATA/NGDEEP/asn_level2/\"\n", + "data_dir_asn_level3 = \"/Users/cpacifici/DATA/NGDEEP/asn_level3/\"\n", + "data_dir_out_image2 = \"/Users/cpacifici/DATA/NGDEEP/image2/\"\n", + "data_dir_out_image3 = \"/Users/cpacifici/DATA/NGDEEP/image3/\"" + ] + }, + { + "cell_type": "markdown", + "id": "77a8ea4e", + "metadata": {}, + "source": [ + "## Read list of rate files for imaging" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d576fd0", + "metadata": {}, + "outputs": [], + "source": [ + "listrate_file = './list_ngdeep_rate.csv'\n", + "listrate = Table.read(listrate_file)\n", + "listrate[np.where(listrate['FILTER']=='CLEAR')]" + ] + }, + { + "cell_type": "markdown", + "id": "0ac742e8", + "metadata": {}, + "source": [ + "## Run image2 on all CLEAR images using the provided association files" + ] + }, + { + "cell_type": "markdown", + "id": "82eb2c93", + "metadata": {}, + "source": [ + "### Check an association file for level 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74bc920c", + "metadata": {}, + "outputs": [], + "source": [ + "asnfile = data_dir_asn_level2 + 'jw02079-o004_20230622t175524_image2_00001_asn.json'\n", + "asn_data = json.load(open(asnfile))\n", + "print(asn_data['products'][0]['members'][0]['expname'])\n", + "asn_data" + ] + }, + { + "cell_type": "markdown", + "id": "3cf2e472", + "metadata": {}, + "source": [ + "**Developer note:** I do not see a way to tell which association file looks at which rate file without opening it and I believe association files and rate files need to be in the same directory where the pipeline is running, so I am going to temporarily copy all association file and all rate files in here and delete them when image2 is done." + ] + }, + { + "cell_type": "markdown", + "id": "0421c67b", + "metadata": {}, + "source": [ + "### Move all needed files to the current directory" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb0a1344", + "metadata": {}, + "outputs": [], + "source": [ + "asn_image2 = glob.glob(data_dir_asn_level2+'*image2*')\n", + "for file in asn_image2:\n", + " shutil.copy(file, './')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c62185ff", + "metadata": {}, + "outputs": [], + "source": [ + "rate_clear = listrate[np.where(listrate['FILTER']=='CLEAR')]['FILENAME']\n", + "for file in rate_clear:\n", + " shutil.copy(data_dir_in+file, './')" + ] + }, + { + "cell_type": "markdown", + "id": "eaa48cf9", + "metadata": {}, + "source": [ + "### Run image2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "373ccb5a", + "metadata": {}, + "outputs": [], + "source": [ + "asn_image2_local = glob.glob('./jw02079-o004_*_image2_*_asn.json')\n", + "asn_image2_local" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dff099be", + "metadata": {}, + "outputs": [], + "source": [ + "for file in asn_image2_local:\n", + " print(file)\n", + " image2 = Image2Pipeline()\n", + " result = image2.call(file,\n", + " save_results=True,\n", + " output_dir=data_dir_out_image2) # Use the defaults for now" + ] + }, + { + "cell_type": "markdown", + "id": "1bddeaa8", + "metadata": {}, + "source": [ + "### Delete association and rate files from local directory" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7caa5b04", + "metadata": {}, + "outputs": [], + "source": [ + "dir_to_be_del = './files_to_be_deleted'\n", + "if not os.path.exists(dir_to_be_del):\n", + " os.mkdir(dir_to_be_del)\n", + " \n", + "for file in asn_image2_local:\n", + " shutil.move(file, dir_to_be_del)\n", + " \n", + "for file in glob.glob('./*rate.fits'):\n", + " shutil.move(file, dir_to_be_del)\n", + " \n", + "shutil.rmtree(dir_to_be_del)" + ] + }, + { + "cell_type": "markdown", + "id": "fe1ae678", + "metadata": {}, + "source": [ + "## Run image3 for the three filters and create catalogs" + ] + }, + { + "cell_type": "markdown", + "id": "e1fd5c10", + "metadata": {}, + "source": [ + "### Check an association file for level 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "add4839e", + "metadata": {}, + "outputs": [], + "source": [ + "asnfile = data_dir_asn_level3 + 'jw02079-o004_20230622t175524_image3_00002_asn.json'\n", + "asn_data = json.load(open(asnfile))\n", + "asn_data" + ] + }, + { + "cell_type": "markdown", + "id": "2f4dd3b2", + "metadata": {}, + "source": [ + "### Move all needed files to the current directory" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97e5a055", + "metadata": {}, + "outputs": [], + "source": [ + "asn_image3 = glob.glob(data_dir_asn_level3+'*image3*')\n", + "for file in asn_image3:\n", + " shutil.copy(file, './')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "faddcfe7", + "metadata": {}, + "outputs": [], + "source": [ + "cal_clear = glob.glob(data_dir_out_image2+'*')\n", + "for file in cal_clear:\n", + " shutil.copy(file, './')" + ] + }, + { + "cell_type": "markdown", + "id": "0cab171d", + "metadata": {}, + "source": [ + "### Run image3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6cf900af", + "metadata": {}, + "outputs": [], + "source": [ + "asn_image3_local = glob.glob('./*image3*.json')\n", + "asn_image3_local" + ] + }, + { + "cell_type": "markdown", + "id": "eaabda11", + "metadata": {}, + "source": [ + "I'll first run the step with the default options, check the catalog and rerun with adjusted parameters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f2f7f0f", + "metadata": {}, + "outputs": [], + "source": [ + "for file in asn_image3_local:\n", + " print(file)\n", + " image3 = Image3Pipeline()\n", + " result = image3.call(file,\n", + " steps={\n", + " 'source_catalog':{'kernel_fwhm':5.0,\n", + " 'snr_threshold':10.0,\n", + " 'npixels':50,\n", + " 'deblend':True,\n", + " },\n", + " 'tweakreg':{'snr_threshold':20,\n", + " 'abs_refcat':'GAIADR2',\n", + " 'save_catalogs':True,\n", + " 'searchrad':3.0,\n", + " 'kernel_fwhm':2.302,\n", + " 'fitgeometry':'shift',\n", + " },\n", + " },\n", + " save_results=True,\n", + " output_dir=data_dir_out_image3)" + ] + }, + { + "cell_type": "markdown", + "id": "e7618691", + "metadata": {}, + "source": [ + "### Inspect the catalogs used for tweakreg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64bf95eb", + "metadata": {}, + "outputs": [], + "source": [ + "cal_files = glob.glob(data_dir_out_image2+'*')\n", + "cat_files = []\n", + "for file in cal_files:\n", + " filename = file.split('/')[-1]\n", + " catname = filename.replace('cal.fits', 'cal_cat.ecsv')\n", + " cat_files.append(catname)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8830aadb", + "metadata": {}, + "outputs": [], + "source": [ + "calexposure = 2\n", + "imviz_cal = Imviz()\n", + "viewer = imviz_cal.default_viewer\n", + "imviz_cal.load_data(cal_files[calexposure])\n", + "plotopt = imviz_cal.plugins['Plot Options']\n", + "plotopt.stretch_preset = '99%'\n", + "imviz_cal.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2ce2462", + "metadata": {}, + "outputs": [], + "source": [ + "catalog = Table.read(cat_files[calexposure])\n", + "t_xy = Table({'x': catalog['x'],\n", + " 'y': catalog['y']})\n", + "\n", + "viewer.marker = {'color': 'orange', 'alpha': 1, 'markersize': 20, 'fill': False}\n", + "viewer.add_markers(t_xy)" + ] + }, + { + "cell_type": "markdown", + "id": "70e6116a", + "metadata": {}, + "source": [ + "### Delete not-needed files from local directory\n", + "\n", + "List includes: cal, outlier_i2d, cal_cat, and asn." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5df25051", + "metadata": {}, + "outputs": [], + "source": [ + "dir_to_be_del = './files_to_be_deleted'\n", + "if not os.path.exists(dir_to_be_del):\n", + " os.mkdir(dir_to_be_del)\n", + " \n", + "list_files = glob.glob('./jw02079*')\n", + "\n", + "print(\"CHECK that list is ok before continuing\")\n", + "list_files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c70a03f6", + "metadata": {}, + "outputs": [], + "source": [ + "for file in list_files:\n", + " shutil.move(file, dir_to_be_del)\n", + "\n", + "shutil.rmtree(dir_to_be_del)" + ] + }, + { + "cell_type": "markdown", + "id": "bc3459a5", + "metadata": {}, + "source": [ + "## Visualize catalog and i2d images in Imviz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c688aa3b", + "metadata": {}, + "outputs": [], + "source": [ + "image3_i2d = [data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f115w_i2d.fits',\n", + " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f150w_i2d.fits',\n", + " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f200w_i2d.fits']\n", + "\n", + "image3_segm = [data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f115w_segm.fits',\n", + " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f150w_segm.fits',\n", + " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f200w_segm.fits']\n", + "\n", + "image3_cat = [data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv',\n", + " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f150w_cat.ecsv',\n", + " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f200w_cat.ecsv']\n", + "\n", + "image3_label = ['F115W',\n", + " 'F150W',\n", + " 'F200W']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7bec8a55", + "metadata": {}, + "outputs": [], + "source": [ + "imviz = Imviz()\n", + "for ii in range(len(image3_i2d)):\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter('ignore')\n", + " imviz.load_data(image3_i2d[ii], data_label=image3_label[ii])\n", + "imviz.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98bcbaf8", + "metadata": {}, + "outputs": [], + "source": [ + "viewer = imviz.default_viewer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad8d9c26", + "metadata": {}, + "outputs": [], + "source": [ + "linking = imviz.plugins['Links Control']\n", + "linking.link_type = 'WCS'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8369edea", + "metadata": {}, + "outputs": [], + "source": [ + "plotopt = imviz.plugins['Plot Options']\n", + "plotopt.multiselect = True\n", + "plotopt.select_all(viewers=True, layers=True)\n", + "plotopt.stretch_preset = '99.5%'\n", + "plotopt.multiselect = False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "235aa97d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb b/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb new file mode 100644 index 000000000..85d1e7fa9 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb @@ -0,0 +1,102 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "5e3e6ab1", + "metadata": {}, + "source": [ + "# Create catalog for spectral extraction\n", + "\n", + "Pick a few interesting objects and create catalogs for each filter with matching IDs for easy analysis after the extraction of the 2D spectra." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9602eab", + "metadata": {}, + "outputs": [], + "source": [ + "from astropy.table import Table\n", + "from jdaviz import Imviz" + ] + }, + { + "cell_type": "markdown", + "id": "aa194c49", + "metadata": {}, + "source": [ + "## Read catalogs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfadc316", + "metadata": {}, + "outputs": [], + "source": [ + "data_dir = '/Users/cpacifici/DATA/NGDEEP/image3/'\n", + "file_f115w = data_dir + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv'\n", + "file_f150w = data_dir + 'jw02079-o004_t001_niriss_clear-f150w_cat.ecsv'\n", + "file_f150w = data_dir + 'jw02079-o004_t001_niriss_clear-f200w_cat.ecsv'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d711d91", + "metadata": {}, + "outputs": [], + "source": [ + "cat_f115w = Table.read(file_f115w)\n", + "cat_f115w" + ] + }, + { + "cell_type": "markdown", + "id": "b44f4b06", + "metadata": {}, + "source": [ + "## Filter interesting objects" + ] + }, + { + "cell_type": "markdown", + "id": "3da1408d", + "metadata": {}, + "source": [ + "## Rewrite catalogs with matching IDs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f54b03d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb new file mode 100644 index 000000000..a2f26e7c8 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6d674dc1", + "metadata": {}, + "source": [ + "# Run spec2 pipeline on rate files" + ] + }, + { + "cell_type": "markdown", + "id": "b28e541b", + "metadata": {}, + "source": [ + "## Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "453e9031", + "metadata": {}, + "outputs": [], + "source": [ + "# Packages that allow us to get information about objects:\n", + "import asdf\n", + "import os\n", + "# Update to a path in your system (see details below at \"Reference files\")\n", + "os.environ[\"CRDS_PATH\"] = \"~/crds_cache_may2023/\"\n", + "os.environ[\"CRDS_SERVER_URL\"] = \"https://jwst-crds.stsci.edu\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9443f2ff", + "metadata": {}, + "outputs": [], + "source": [ + "# The calwebb_spec and spec3 pipelines\n", + "from jwst.pipeline import Spec2Pipeline\n", + "from jwst.pipeline import Spec3Pipeline\n", + "\n", + "# individual steps\n", + "from jwst.assign_wcs import AssignWcsStep\n", + "from jwst.assign_wcs import nirspec\n", + "from jwst.background import BackgroundStep\n", + "from jwst.imprint import ImprintStep\n", + "from jwst.msaflagopen import MSAFlagOpenStep\n", + "from jwst.extract_2d import Extract2dStep\n", + "from jwst.srctype import SourceTypeStep\n", + "from jwst.wavecorr import WavecorrStep\n", + "from jwst.flatfield import FlatFieldStep\n", + "from jwst.pathloss import PathLossStep\n", + "from jwst.photom import PhotomStep\n", + "from jwst.cube_build import CubeBuildStep\n", + "from jwst.extract_1d import Extract1dStep\n", + "from jwst.combine_1d import Combine1dStep\n", + "\n", + "# data models\n", + "from jwst import datamodels\n", + "\n", + "# associations\n", + "from jwst.associations import asn_from_list\n", + "from jwst.associations.lib.rules_level3_base import DMS_Level3_Base\n", + "\n", + "# astropy\n", + "from astropy.io import fits\n", + "from astropy.utils.data import download_file\n", + "import astropy.units as u\n", + "from astropy import wcs\n", + "from astropy.wcs import WCS\n", + "from astropy.visualization import ImageNormalize, ManualInterval, LogStretch, LinearStretch, AsinhStretch\n", + "\n", + "from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87cdec65", + "metadata": {}, + "outputs": [], + "source": [ + "import jwst\n", + "print('jwst',jwst.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "67a46e39", + "metadata": {}, + "source": [ + "## Files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "353f52f4", + "metadata": {}, + "outputs": [], + "source": [ + "association_level2 = './jw02079-o004_f115w_f150wcat_asn.json'\n", + "rate = './jw02079004001_05101_00001_nis_rate.fits'\n", + "\n", + "#association_level2 = './jw02079-o004_f150w_f150wcat_asn.json'\n", + "#rate = './jw02079004002_11101_00001_nis_rate.fits'\n", + "\n", + "#association_level2 = './jw02079-o004_f200w_f150wcat_asn.json'\n", + "#rate = './jw02079004003_05101_00001_nis_rate.fits'\n" + ] + }, + { + "cell_type": "markdown", + "id": "d71ccaa3", + "metadata": {}, + "source": [ + "## Run spec2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2d5b653", + "metadata": {}, + "outputs": [], + "source": [ + "output_dir = './pipeline_output/'\n", + "# Create an instance of the pipeline class\n", + "spec2 = Spec2Pipeline()\n", + "\n", + "# Set some parameters that pertain to the entire pipeline\n", + "#spec2.save_results = True\n", + "#spec2.output_dir = output_dir\n", + "\n", + "# Here you could some parameters that pertain to some of the individual steps\n", + "#spec2.extract_2d.mmag_extract = 20 # Minimum magnitude of objects to extract for WFSS data\n", + "#spec2.extract_1d.bkg_fit = 'poly' # Fit a polynomial to the background values for each column or row\n", + "#spec2.bkg_subtract.skip = True # Skip the default image-from-image background subtraction method\n", + "\n", + "# Call the call() method, using an input association file.\n", + "# Call method reinitialized the default values, so parameters need to be changed inside the call\n", + "result = spec2.call(association_level2, \n", + " steps={'bkg_subtract':{'skip':True},\n", + " 'extract_2d':{'wfss_nbright':500,\n", + " 'wfss_extract_half_height': 25,\n", + " },\n", + " 'photom':{'skip':True},\n", + " },\n", + " save_results=True,\n", + " output_dir=output_dir)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41661aec", + "metadata": {}, + "outputs": [], + "source": [ + "calfile = './pipeline_output/jw02079004001_05101_00001_nis_cal.fits'\n", + "calimage = fits.open(calfile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1eb9efd", + "metadata": {}, + "outputs": [], + "source": [ + "source = 4\n", + "print(calimage['SCI',source].header['SOURCEID'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46fe0a2d", + "metadata": {}, + "outputs": [], + "source": [ + "data = calimage['SCI',source].data\n", + "plt.imshow(data, origin='lower', vmin=0.4, vmax=0.8)\n", + "plt.colorbar()" + ] + }, + { + "cell_type": "markdown", + "id": "197cfb65", + "metadata": {}, + "source": [ + "There is no background subtraction so this is absolutely useless." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7c09858", + "metadata": {}, + "outputs": [], + "source": [ + "x1dfile = './pipeline_output/jw02079004002_11101_00001_nis_x1d.fits'\n", + "x1dspec = fits.open(x1dfile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "930ac35b", + "metadata": {}, + "outputs": [], + "source": [ + "data = x1dspec['EXTRACT1D',source].data\n", + "plt.plot(data['WAVELENGTH'], data['FLUX'])\n", + "plt.xlabel('wavelength (um)')\n", + "plt.ylabel('flux (adu)')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "353bad8c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/07_specviz2d_extract1d.ipynb b/notebooks/NIRISS_WFSS_advanced/07_specviz2d_extract1d.ipynb new file mode 100644 index 000000000..3f7c330e6 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/07_specviz2d_extract1d.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "e3b32f0d", + "metadata": {}, + "outputs": [], + "source": [ + "from jdaviz import Specviz2d, Specviz\n", + "from specutils import Spectrum1D\n", + "import numpy as np\n", + "from astropy.io import fits\n", + "from matplotlib import pyplot as plt\n", + "from astropy import units as u" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7d71a35", + "metadata": {}, + "outputs": [], + "source": [ + "calfile = './runpipeline_spec2/pipeline_output/jw02079004001_05101_00001_nis_cal.fits'\n", + "calimage = fits.open(calfile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "486ae51a", + "metadata": {}, + "outputs": [], + "source": [ + "for i in range(1,15):\n", + " print(i, calimage['SCI',i].header['SOURCEID'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f6526475", + "metadata": {}, + "outputs": [], + "source": [ + "source = 3\n", + "print(calimage['SCI',source].header['SOURCEID'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c637d66", + "metadata": {}, + "outputs": [], + "source": [ + "data = calimage['SCI',source].data\n", + "data = np.nan_to_num(data)\n", + "invertdata = np.flip(data, 1)\n", + "wave = calimage['WAVELENGTH',source].data[0]\n", + "invertwave = np.flip(wave)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b7c0590", + "metadata": {}, + "outputs": [], + "source": [ + "spec2d = Spectrum1D(flux=invertdata*u.Jy, spectral_axis=invertwave*u.um)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b91e580", + "metadata": {}, + "outputs": [], + "source": [ + "specviz2d = Specviz2d()\n", + "specviz2d.load_data(spec2d)\n", + "specviz2d.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f01a54f", + "metadata": {}, + "outputs": [], + "source": [ + "spec1d_pix = specviz2d.get_data('Spectrum 1D')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "154924cb", + "metadata": {}, + "outputs": [], + "source": [ + "spec1d_f115w = Spectrum1D(flux=spec1d_pix.flux, spectral_axis=invertwave*u.um)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20718648", + "metadata": {}, + "outputs": [], + "source": [ + "spec1d_f150w = Spectrum1D(flux=spec1d_pix.flux, spectral_axis=invertwave*u.um)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2477a732", + "metadata": {}, + "outputs": [], + "source": [ + "spec1d_f200w = Spectrum1D(flux=spec1d_pix.flux, spectral_axis=invertwave*u.um)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a1ab9cf", + "metadata": {}, + "outputs": [], + "source": [ + "specviz = Specviz()\n", + "specviz.load_data(spec1d_f115w, data_label='f115w')\n", + "specviz.load_data(spec1d_f150w, data_label='f150w')\n", + "specviz.load_data(spec1d_f200w, data_label='f200w')\n", + "specviz.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0645e33f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/08_compare_extractions.ipynb b/notebooks/NIRISS_WFSS_advanced/08_compare_extractions.ipynb new file mode 100755 index 000000000..e0cc02bb9 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/08_compare_extractions.ipynb @@ -0,0 +1,962 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# WFSS Spectra Part 0: Optimal Extraction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Use case:** optimal extraction of grism spectra; redshift measurement; emission-line maps. Simplified version of [JDox Science Use Case # 33](https://jwst-docs.stsci.edu/near-infrared-imager-and-slitless-spectrograph/niriss-example-science-programs/niriss-wfss-with-nircam-parallel-imaging-of-galaxies-in-lensing-clusters).
\n", + "**Data:** JWST simulated NIRISS images from [MIRAGE](https://jwst-docs.stsci.edu/jwst-other-tools/mirage-data-simulator), run through the [JWST calibration pipeline](https://jwst-pipeline.readthedocs.io/en/latest/); galaxy cluster.
\n", + "**Tools:** specutils, astropy, pandas, emcee, lmfit, corner, h5py.
\n", + "**Cross-intrument:** NIRSpec
\n", + "**Documentation:** This notebook is part of a STScI's larger [post-pipeline Data Analysis Tools Ecosystem](https://jwst-docs.stsci.edu/jwst-post-pipeline-data-analysis).
\n", + "\n", + "\n", + "## Introduction\n", + "\n", + "This notebook is 1 of 4 in a set focusing on NIRISS WFSS data:\n", + " 1. 1D optimal extraction since the JWST pipeline only provides a box extraction. Optimal extraction improves S/N of spectra for faint sources.\n", + " 2. Combine and normalize 1D spectra.\n", + " 3. Cross correlate galaxy with template to get redshift.\n", + " 4. Spatially resolved emission line map.\n", + "\n", + "This notebook will start with [post-pipeline](https://jwst-docs.stsci.edu/jwst-data-reduction-pipeline) products of NIRISS WFSS, 2D rectified spectra, from spec level3.\n", + "\n", + "Optimal extraction requires source morphology along the cross-dispersion direction, which will be retrieved from direct images taken along with WFSS observations. Morphology along dispersion direction is also essential to infer the spectral resolution, which will be obtained using template fitting to get redshift and stellar population in notebook #3 of this set.\n", + "\n", + "**Note:** We here assume reduction of the 2D rectified spectrum has been performed at a decent level, i.e. there is no contaminating flux from other sources on the target 2D spectrum, and that background is already subtracted." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "from scipy.ndimage import rotate\n", + "from scipy.optimize import curve_fit\n", + "\n", + "\n", + "from astropy.convolution import Gaussian2DKernel\n", + "from astropy.io import fits\n", + "from astropy.stats import gaussian_fwhm_to_sigma\n", + "from astropy.table import QTable\n", + "import astropy.units as u\n", + "from astropy.visualization import make_lupton_rgb, SqrtStretch, ImageNormalize, simple_norm\n", + "import astropy.wcs as wcs\n", + "from astropy.io import ascii\n", + "\n", + "from specutils import Spectrum1D\n", + "from astropy.nddata import StdDevUncertainty\n", + "\n", + "import specutils\n", + "print('specutils', specutils.__version__)\n", + "\n", + "import astropy\n", + "print('astropy', astropy.__version__)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The version should be \n", + "- specutils 1.0\n", + "- astropy 4.0.1.post1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import matplotlib as mpl\n", + "\n", + "mpl.rcParams['savefig.dpi'] = 80\n", + "mpl.rcParams['figure.dpi'] = 80" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0.Download and load data:\n", + "These include pipeline processed data for NIRISS, as well as photometric catalog from image step3." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.exists('./pipeline_products'):\n", + " import zipfile\n", + " import urllib.request\n", + " boxlink = 'https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/NIRISS_lensing_cluster/pipeline_products.zip'\n", + " boxfile = './pipeline_products.zip'\n", + " urllib.request.urlretrieve(boxlink, boxfile)\n", + " zf = zipfile.ZipFile(boxfile, 'r')\n", + " zf.extractall()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DIR_DATA = './pipeline_products/'\n", + "\n", + "# Output directory;\n", + "DIR_OUT = './output/'\n", + "if not os.path.exists(DIR_OUT):\n", + " os.mkdir(DIR_OUT)\n", + "\n", + "# Filter for detection and science image;\n", + "filt_det = 'f200w'\n", + "\n", + "# Image array from direct image. This is for optimal extraction and masking.\n", + "# This image should already be sky-subtracted; otherwise, you will encounter a wrong result with optimal extraction.\n", + "infile = '%sl3_nis_%s_i2d_skysub.fits'%(DIR_DATA,filt_det)\n", + "hdu = fits.open(infile)\n", + "\n", + "# This is just for error array;\n", + "infile = '%sl3_nis_%s_i2d.fits'%(DIR_DATA,filt_det)\n", + "hdu_err = fits.open(infile)\n", + "\n", + "data = hdu[0].data\n", + "imwcs = wcs.WCS(hdu[0].header, hdu)\n", + "\n", + "err = hdu_err[2].data\n", + "weight = 1/np.square(err)\n", + "\n", + "# Segmentation map;\n", + "# This can be prepared by running Photutils, if the pipeline does not generate one.\n", + "segfile = '%sl3_nis_%s_i2d_seg.fits'%(DIR_DATA, filt_det)\n", + "seghdu = fits.open(segfile)\n", + "segdata = seghdu[0].data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load catalog from image level3;\n", + "# to obtain source position in pixel coordinate.\n", + "catfile = '%sl3_nis_%s_cat.ecsv'%(DIR_DATA, filt_det)\n", + "fd = ascii.read('%s'%catfile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Make a broadband flux catalog.\n", + "- For a convenient reason, here we compile catalogs into a flux catalog, which will be used in the following notebook (01b).\n", + "- To run this cell, you will need a photometric catalog of sources, that list sources position and flux for each filter. For now, I use this catalog prepared in another notebook. (\"sources_extend_01.cat\")\n", + "- This catalog can also be used for generic phot-z/SED fitting softwares, like EAZY and gsf (see notebook No.04).\n", + "\n", + "#### For now, we use an input catalog." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filts = ['f115w', 'f150w', 'f200w', 'f090w', 'f435w', 'f606w', 'f814w', 'f105w', 'f125w', 'f140w', 'f160w']\n", + "eazy_filts = [309, 310, 311, 308, 1, 4, 6, 202, 203, 204, 205]\n", + "magzp = 25.0 # magnitude zeropoint in the catalog.\n", + "\n", + "# Read catalog;\n", + "fd_input = ascii.read('%ssources_extend_01.cat'%(DIR_DATA))\n", + "ra_input = fd_input['x_or_RA']\n", + "dec_input = fd_input['y_or_Dec']\n", + "\n", + "# Header;\n", + "fw = open('%sl3_nis_flux.cat'%(DIR_OUT), 'w')\n", + "fw.write('# id')\n", + "for ff in range(len(filts)):\n", + " fw.write(' F%d E%d'%(eazy_filts[ff], eazy_filts[ff]))\n", + "fw.write('\\n')\n", + "\n", + "# Contents;\n", + "for ii in range(len(fd['id'])):\n", + " \n", + " rtmp = np.sqrt((fd['sky_centroid'].ra.value[ii] - ra_input[:])**2 + (fd['sky_centroid'].dec.value[ii] - dec_input[:])**2)\n", + " iix = np.argmin(rtmp)\n", + " \n", + " for ff in range(len(filts)):\n", + " if ff == 0:\n", + " fw.write('%d'%(fd['id'][ii]))\n", + "\n", + " mag = fd_input['niriss_%s_magnitude'%filts[ff]][iix]\n", + " flux_nu = 10**((mag-magzp)/(-2.5))\n", + "\n", + " # Currently, the catalog does not provide proper error;\n", + " # Assuming 5% error for flux.\n", + " \n", + " flux_err_nu = flux_nu * 0.05\n", + "\n", + " fw.write(' %.5e %.5e'%(flux_nu, flux_err_nu))\n", + "\n", + " fw.write('\\n')\n", + "fw.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1.Load 2D spectrum;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Which filter, grating, and object?\n", + "filt = 'f200w'\n", + "\n", + "#grism = 'G150R'\n", + "grism = 'G150C'\n", + "\n", + "id = '00004'\n", + "\n", + "# Zero-indexed number for dither --- the test data here has two dither positions, so 0 or 1.\n", + "ndither = 0\n", + "\n", + "file_2d = '%sl3_nis_%s_%s_s%s_cal.fits'%(DIR_DATA, filt, grism, id)\n", + "hdu_2d = fits.open(file_2d)\n", + "\n", + "# Align grism direction\n", + "# - x-direction = Dispersion (wavelength) direction.\n", + "# - y-direction = Cross-dispersion.\n", + "# in this notebook.\n", + " \n", + "if grism == 'G150C':\n", + " # If spectrum is horizontal;\n", + " data_2d = hdu_2d[ndither*7+1].data\n", + " dq_2d = hdu_2d[ndither*7+2].data\n", + " err_2d = hdu_2d[ndither*7+3].data\n", + " wave_2d = hdu_2d[ndither*7+4].data\n", + "else:\n", + " data_2d = rotate(hdu_2d[ndither*7+1].data, 90)\n", + " dq_2d = rotate(hdu_2d[ndither*7+2].data, 90)\n", + " err_2d = rotate(hdu_2d[ndither*7+3].data, 90)\n", + " wave_2d = rotate(hdu_2d[ndither*7+4].data, 90)\n", + "\n", + "# Get position angle of observation;\n", + "hd_2d = hdu_2d[1].header\n", + "PA_V3 = hd_2d['PA_V3']\n", + "PA_V3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.imshow(data_2d, vmin=0, vmax=1000)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get light profile of the source;\n", + "\n", + "# Again, y is for cross-dispersion, and x is for dispersion directions.\n", + "y2d,x2d = data_2d.shape[:]\n", + "\n", + "# Cut out segmentation map;\n", + "iix = np.where(fd['id']==int(id))[0][0]\n", + "\n", + "# Target position from image 3 catalog;\n", + "ycen = fd['ycentroid'][iix].value\n", + "xcen = fd['xcentroid'][iix].value\n", + "\n", + "# Cutout size = y direction of 2D spectrum;\n", + "rsq = y2d\n", + "\n", + "sci_cut = data[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", + "seg_cut = segdata[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", + "\n", + "# Rotate image for PA of Grism observation;\n", + "if grism == 'G150C':\n", + " sci_rot = rotate(sci_cut, PA_V3)\n", + "else:\n", + " sci_rot = rotate(sci_cut, PA_V3+90)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### WFSS grism is dispersed in a direction of x-axis in the plot below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.imshow(sci_rot, vmin=0, vmax=1.0)\n", + "plt.title('Direct image')\n", + "plt.xlabel('Wavelength direction >>>', color='r', fontsize=18)\n", + "plt.ylabel('Cross-dispersion direction >>>', color='r', fontsize=18)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2.Get light profile at different x position --- This will be used for optimal extraction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ii in range(sci_rot.shape[1]):\n", + " flux_tmp = sci_rot[:,ii]\n", + " xx_tmp = np.arange(0, len(sci_rot[:,ii]), 1)\n", + " plt.plot(xx_tmp, flux_tmp, label='x=%d'%ii)\n", + "plt.legend(loc=0, fontsize=8)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sum along x (disperse) direction\n", + "flux_y = np.zeros(len(sci_rot[:,0]), 'float')\n", + "for ii in range(sci_rot.shape[0]):\n", + " flux_y[ii] = np.sum(sci_rot[ii,:])\n", + "\n", + "# Sky subtraction, if needed.\n", + "#sky = np.mean([flux_y[0], flux_y[-1]])\n", + "\n", + "# Normalize;\n", + "flux_y[:] /= flux_y.sum()\n", + "\n", + "plt.plot(xx_tmp, flux_y)\n", + "plt.xlabel('y-position')\n", + "plt.ylabel('Source Flux')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3.One-dimensional extraction;" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Show pipeline 1D extraction as an example;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Normal extraction;\n", + "flux_disp1 = np.zeros(x2d, 'float')\n", + "err_disp1 = np.zeros(x2d, 'float')\n", + "wave_disp1 = np.zeros(x2d, 'float')\n", + " \n", + "for ii in range(x2d): # Wavelength direction.\n", + " mask_tmp = (dq_2d[:,ii] == 0) & (err_2d[:,ii]>0)\n", + "\n", + " # Sum within a box;\n", + " flux_disp1[ii] = np.sum(data_2d[:,ii][mask_tmp]) \n", + " err_disp1[ii] = np.sqrt(np.sum(err_2d[:,ii][mask_tmp]**2)) \n", + " wave_disp1[ii] = wave_2d[0,ii]\n", + "\n", + "plt.errorbar(wave_disp1, flux_disp1, yerr=err_disp1)\n", + "plt.xlim(1.7,2.3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Optimal extraction;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Following Horne(1986, PASP, 98, 609);\n", + "flux_disp = np.zeros(x2d, 'float')\n", + "err_disp = np.zeros(x2d, 'float')\n", + "wave_disp = np.zeros(x2d, 'float')\n", + "\n", + "# Sigma clipping.\n", + "sig = 5.0\n", + "\n", + "for ii in range(x2d): # ii : wavelength element.\n", + " # Mask; \n", + " # 1. DQ array\n", + " # 2. error value\n", + " # 3. CR detection\n", + " mask_tmp = (dq_2d[:,ii] == 0) & (err_2d[:,ii]>0) & ( (data_2d[:,ii] - flux_y[:] * flux_disp1[ii])**2 < sig**2 * err_2d[:,ii]**2)\n", + " ivar = 1. / err_2d[:,ii]**2\n", + "\n", + " num = flux_y[:] * data_2d[:,ii] * ivar\n", + " den = flux_y[:]**2 * ivar\n", + " flux_disp[ii] = num[mask_tmp].sum(axis=0) / den[mask_tmp].sum(axis=0)\n", + " err_disp[ii] = np.sqrt(1./den[mask_tmp].sum(axis=0))\n", + " wave_disp[ii] = wave_2d[0,ii]\n", + " \n", + "plt.errorbar(wave_disp, flux_disp, yerr=err_disp)\n", + "plt.xlim(1.7,2.3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Compare;\n", + "plt.errorbar(wave_disp, flux_disp, yerr=err_disp, color='r', label='Optimal')\n", + "plt.errorbar(wave_disp1, flux_disp1, yerr=err_disp1, color='b', alpha=0.5, label='Box')\n", + "plt.ylim(-10, 20000)\n", + "plt.legend(loc=0)\n", + "plt.xlabel('Wavelength')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4.Write 1d spectrum out to a file;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "file_1d = '%sl3_nis_%s_%s_s%s_1d_opt.fits'%(DIR_OUT, filt, grism, id)\n", + "\n", + "# Now make it into a Spectrum1D instance.\n", + "obs = Spectrum1D(spectral_axis=wave_disp*u.um,\n", + " flux=flux_disp*u.MJy,\n", + " uncertainty=StdDevUncertainty(err_disp), unit='MJy')\n", + "obs.write(file_1d, format='tabular-fits', overwrite=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5.Light profile along x-axis = Resolution of dispersed spectrum;\n", + "As WFSS does not have a slit, any dispersed spectrum is affected by source morphology. The estimate on the effective spectral resolution will be needed in the following notebook. And we here try to estimate it beforehand;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ii in range(sci_rot.shape[0]):\n", + " flux_tmp = sci_rot[ii,:]\n", + " xx_tmp = np.arange(0, len(sci_rot[ii,:]), 1)\n", + " plt.plot(xx_tmp, flux_tmp, label='y=%d'%(ii))\n", + " \n", + "plt.legend(loc=1, fontsize=8)\n", + "plt.xlabel('Wavelength direction')\n", + "plt.title('Source light profile along dispersed direction', fontsize=14)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### *Unless you are interested in spatially resolved spectra, you can stack and get light profile as a good approximation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sum along cross-disperse direction\n", + "flux_x = np.zeros(len(sci_rot[0,:]), 'float')\n", + "for ii in range(sci_rot.shape[0]):\n", + " flux_x[ii] = np.sum(sci_rot[:,ii])\n", + "\n", + "# Normalize;\n", + "flux_x[:] /= flux_x.sum()\n", + "\n", + "plt.plot(xx_tmp, flux_x, label='Convolution kernel')\n", + "plt.legend(loc=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Fit with a moffat function;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Fitting function with Moffat\n", + "\n", + "# Moffat fnc.\n", + "def moffat(xx, A, x0, gamma, alp):\n", + " yy = A * (1. + (xx-x0)**2/gamma**2)**(-alp)\n", + " return yy\n", + "\n", + "def fit_mof(xx, lsf):\n", + " #xx = lsf * 0\n", + " #for ii in range(len(lsf)):\n", + " # xx[ii] = ii - len(lsf)/2.\n", + " popt, pcov = curve_fit(moffat, xx, lsf)\n", + " return popt\n", + "\n", + "def LSF_mof(xsf, lsf, f_plot=True):\n", + " '''\n", + " Input:\n", + " =======\n", + " xsf : x axis for the profile.\n", + " lsf : light profile. \n", + " '''\n", + " \n", + " #for ii in range(len(sci[0,:])):\n", + " # lsf[ii] = np.mean(sci_rot[int(height/2.-5):int(height/2.+5), ii])\n", + " # xsf[ii] = ii - len(lsf)/2.\n", + "\n", + " try:\n", + " A, xm, gamma, alpha = fit_mof(xsf, lsf)\n", + " except RuntimeError:\n", + " print('Fitting failed.')\n", + " A, xm, gamma, alpha = -1, -1, -1, -1\n", + " pass\n", + "\n", + " if A>0:\n", + " lsf_mod = moffat(xsf, A, 0, gamma, alpha)\n", + " \n", + " if f_plot:\n", + " yy = moffat(xsf, A, xm, gamma, alpha)\n", + " plt.plot(xsf, yy, 'r.', ls='-', label='Data')\n", + " plt.plot(xsf, lsf_mod, 'b+', ls='-', label='Model:$gamma=%.2f$\\n$alpha=%.2f$'%(gamma, alpha))\n", + " plt.legend()\n", + " plt.show()\n", + " \n", + " return A, xm, gamma, alpha" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# LSF, line spread function\n", + "iix_peak = np.argmax(flux_x)\n", + "xx_tmp_shift = xx_tmp - xx_tmp[iix_peak]\n", + "A, xm, gamma, alpha = LSF_mof(xx_tmp_shift, flux_x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Write it down;\n", + "# Tha parameters are in unit of pixel.\n", + "fm = open('%sl3_nis_%s_%s_s%s_moffat.txt'%(DIR_OUT, filt, grism, id), 'w')\n", + "fm.write('# A x0 gamma alp\\n')\n", + "fm.write('# Moffat function\\n')\n", + "fm.write('%.3f %.3f %.3f %.3f\\n'%(A, xm, gamma, alpha))\n", + "\n", + "fm.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Repeat for other filters, other objects.\n", + "### The following big colum executes the same processes above for other filters and dither position." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "grism = 'G150C'\n", + "id = '00004'\n", + "DIR_OUT = './output/'\n", + "if not os.path.exists(DIR_OUT):\n", + " os.mkdir(DIR_OUT)\n", + "\n", + "filts = ['f115w', 'f150w', 'f200w']\n", + "ndithers = np.arange(0, 2, 1)\n", + "\n", + "sig = 5.0\n", + "\n", + "for filt in filts:\n", + " print(filt)\n", + " \n", + " # 2d spectrum;\n", + " file_2d = '%sl3_nis_%s_%s_s%s_cal.fits'%(DIR_DATA, filt, grism, id)\n", + " hdu_2d = fits.open(file_2d)\n", + "\n", + " for ndither in ndithers:\n", + " print(ndither)\n", + "\n", + " if grism == 'G150C':\n", + " # If spectrum is horizontal;\n", + " data_2d = hdu_2d[ndither*7+1].data\n", + " dq_2d = hdu_2d[ndither*7+2].data\n", + " err_2d = hdu_2d[ndither*7+3].data\n", + " wave_2d = hdu_2d[ndither*7+4].data\n", + " else:\n", + " data_2d = rotate(hdu_2d[ndither*7+1].data, 90)\n", + " dq_2d = rotate(hdu_2d[ndither*7+2].data, 90)\n", + " err_2d = rotate(hdu_2d[ndither*7+3].data, 90)\n", + " wave_2d = rotate(hdu_2d[ndither*7+4].data, 90)\n", + "\n", + " y2d,x2d = data_2d.shape[:]\n", + "\n", + " plt.close()\n", + " plt.imshow(data_2d, vmin=0, vmax=300)\n", + " plt.show()\n", + "\n", + " # Re-extract 2d image;\n", + " #if ndither == 0:\n", + " rsq = y2d\n", + " sci_cut = data[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", + " seg_cut = segdata[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", + "\n", + " # Not sure if the offset in extractioin box is bug ;\n", + " if grism == 'G150C':\n", + " sci_rot = rotate(sci_cut, PA_V3+0)\n", + " else:\n", + " sci_rot = rotate(sci_cut, PA_V3+0+90)\n", + "\n", + "\n", + " #\n", + " # This is for spectral resolution;\n", + " #\n", + " # Get light profile along the x-axis\n", + " for ii in range(sci_rot.shape[0]):\n", + " flux_tmp = sci_rot[ii,:]\n", + " xx_tmp = np.arange(0, len(sci_rot[ii,:]), 1)\n", + "\n", + " # Sum along cross-disperse direction\n", + " flux_x = np.zeros(len(sci_rot[0,:]), 'float')\n", + " for ii in range(sci_rot.shape[0]):\n", + " flux_x[ii] = np.sum(sci_rot[ii,:])\n", + "\n", + " # Normalize;\n", + " flux_x[:] /= flux_x.sum()\n", + "\n", + " # LSF\n", + " iix_peak = np.argmax(flux_x)\n", + " xx_tmp_shift = xx_tmp - xx_tmp[iix_peak]\n", + " A, xm, gamma, alpha = LSF_mof(xx_tmp_shift, flux_x)\n", + "\n", + " if ndither == 0:\n", + " # Write it down;\n", + " fm = open('%sl3_nis_%s_%s_s%s_moffat.txt'%(DIR_OUT, filt, grism, id), 'w')\n", + " fm.write('# A x0 gamma alp\\n')\n", + " fm.write('# Moffat function\\n')\n", + " fm.write('%.3f %.3f %.3f %.3f\\n'%(A, xm, gamma, alpha))\n", + " fm.close()\n", + "\n", + " #\n", + " # This is for Optimal extraction;\n", + " #\n", + " # Sum along x (disperse) direction\n", + " flux_y = np.zeros(len(sci_rot[:,0]), 'float')\n", + " for ii in range(sci_rot.shape[0]):\n", + " flux_y[ii] = np.sum(sci_rot[ii,:])\n", + " \n", + " # Normalize;\n", + " flux_y[:] /= flux_y.sum()\n", + "\n", + "\n", + " # Following Horne;\n", + " flux_disp = np.zeros(x2d, 'float')\n", + " err_disp = np.zeros(x2d, 'float')\n", + " wave_disp = np.zeros(x2d, 'float')\n", + "\n", + " for ii in range(x2d):\n", + " # Mask; \n", + " # 1. DQ array\n", + " # 2. error value\n", + " # 3. CR detection\n", + " mask_tmp = (dq_2d[:,ii] == 0) & (err_2d[:,ii] > 0)\n", + " ivar = 1. / err_2d[:,ii]**2\n", + "\n", + " num = flux_y[:] * data_2d[:,ii] * ivar \n", + " den = flux_y[:]**2 * ivar\n", + " flux_disp[ii] = num[mask_tmp].sum(axis=0)/den[mask_tmp].sum(axis=0)\n", + " err_disp[ii] = np.sqrt(1./den[mask_tmp].sum(axis=0))\n", + " wave_disp[ii] = wave_2d[0,ii]\n", + "\n", + "\n", + " plt.close()\n", + " con_plot = (wave_disp>0)\n", + " plt.errorbar(wave_disp[con_plot], flux_disp[con_plot], yerr=err_disp[con_plot])\n", + " plt.ylim(-0, 3000)\n", + " plt.show()\n", + "\n", + " # Wirte:\n", + " # Now make it into a Spectrum1D instance.\n", + " file_1d = '%sl3_nis_%s_%s_s%s_ndither%d_1d_opt.fits'%(DIR_OUT, filt, grism, id, ndither)\n", + "\n", + " if wave_disp[1] - wave_disp[0] < 0:\n", + " obs = Spectrum1D(spectral_axis=wave_disp[::-1]*u.um,\n", + " flux=flux_disp[::-1]*u.MJy,\n", + " uncertainty=StdDevUncertainty(err_disp[::-1]), unit='MJy')\n", + " else:\n", + " obs = Spectrum1D(spectral_axis=wave_disp*u.um,\n", + " flux=flux_disp*u.MJy,\n", + " uncertainty=StdDevUncertainty(err_disp), unit='MJy')\n", + " \n", + " obs.write(file_1d, format='tabular-fits', overwrite=True)\n", + " \n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Another object;\n", + "Absorption line galaxy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "grism = 'G150C'\n", + "id = '00003'\n", + "DIR_OUT = './output/'\n", + "\n", + "filts = ['f115w', 'f150w', 'f200w']\n", + "ndithers = np.arange(0,2,1) # There are four dithers in the data set;\n", + "\n", + "sig = 5.0\n", + "\n", + "for filt in filts:\n", + " print(filt)\n", + " # 2d spectrum;\n", + " file_2d = '%sl3_nis_%s_%s_s%s_cal.fits'%(DIR_DATA, filt, grism, id)\n", + " hdu_2d = fits.open(file_2d)\n", + "\n", + " for ndither in ndithers:\n", + " print(ndither)\n", + " if grism == 'G150C':\n", + " # If spectrum is horizontal;\n", + " data_2d = hdu_2d[ndither*7+1].data\n", + " dq_2d = hdu_2d[ndither*7+2].data\n", + " err_2d = hdu_2d[ndither*7+3].data\n", + " wave_2d = hdu_2d[ndither*7+4].data\n", + " else:\n", + " data_2d = rotate(hdu_2d[ndither*7+1].data, 90)\n", + " dq_2d = rotate(hdu_2d[ndither*7+2].data, 90)\n", + " err_2d = rotate(hdu_2d[ndither*7+3].data, 90)\n", + " wave_2d = rotate(hdu_2d[ndither*7+4].data, 90)\n", + "\n", + " y2d,x2d = data_2d.shape[:]\n", + "\n", + " plt.close()\n", + " plt.imshow(data_2d, vmin=0, vmax=20)\n", + " plt.show()\n", + "\n", + " # Re-extract 2d image;\n", + " #if ndither == 0:\n", + " rsq = y2d\n", + " sci_cut = data[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", + " seg_cut = segdata[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", + "\n", + " # Not sure if the offset in extractioin box is bug ;\n", + " if grism == 'G150C':\n", + " sci_rot = rotate(sci_cut, PA_V3+0)\n", + " else:\n", + " sci_rot = rotate(sci_cut, PA_V3+0+90)\n", + "\n", + " #\n", + " # This is for spectral resolution;\n", + " #\n", + " # Get light profile along the x-axis\n", + " for ii in range(sci_rot.shape[0]):\n", + " flux_tmp = sci_rot[ii,:]\n", + " xx_tmp = np.arange(0, len(sci_rot[ii,:]), 1)\n", + "\n", + " # Sum along cross-disperse direction\n", + " flux_x = np.zeros(len(sci_rot[0,:]), 'float')\n", + " for ii in range(sci_rot.shape[0]):\n", + " flux_x[ii] = np.sum(sci_rot[ii,:])\n", + "\n", + " # Normalize;\n", + " flux_x[:] /= flux_x.sum()\n", + "\n", + " # LSF\n", + " iix_peak = np.argmax(flux_x)\n", + " xx_tmp_shift = xx_tmp - xx_tmp[iix_peak]\n", + " A, xm, gamma, alpha = LSF_mof(xx_tmp_shift, flux_x)\n", + "\n", + " if ndither == 0:\n", + " # Write it down;\n", + " fm = open('%sl3_nis_%s_%s_s%s_moffat.txt'%(DIR_OUT, filt, grism, id), 'w')\n", + " fm.write('# A x0 gamma alp\\n')\n", + " fm.write('# Moffat function\\n')\n", + " fm.write('%.3f %.3f %.3f %.3f\\n'%(A, xm, gamma, alpha))\n", + " fm.close()\n", + "\n", + "\n", + " #\n", + " # This is for Optimal extraction;\n", + " #\n", + " # Sum along x (disperse) direction\n", + " flux_y = np.zeros(len(sci_rot[:,0]), 'float')\n", + " for ii in range(sci_rot.shape[0]):\n", + " flux_y[ii] = np.sum(sci_rot[ii,:])\n", + "\n", + " \n", + " # Normalize;\n", + " flux_y[:] /= flux_y.sum()\n", + "\n", + "\n", + " # Following Horne;\n", + " flux_disp = np.zeros(x2d, 'float')\n", + " err_disp = np.zeros(x2d, 'float')\n", + " wave_disp = np.zeros(x2d, 'float')\n", + "\n", + " for ii in range(x2d):\n", + " # Mask; \n", + " # 1. DQ array\n", + " # 2. error value\n", + " # 3. CR detection\n", + " mask_tmp = (dq_2d[:,ii] == 0) & (err_2d[:,ii] > 0) \n", + " ivar = 1. / err_2d[:,ii]**2\n", + "\n", + " num = flux_y[:] * data_2d[:,ii] * ivar \n", + " den = flux_y[:]**2 * ivar\n", + " flux_disp[ii] = num[mask_tmp].sum(axis=0)/den[mask_tmp].sum(axis=0)\n", + " err_disp[ii] = np.sqrt(1./den[mask_tmp].sum(axis=0))\n", + " wave_disp[ii] = wave_2d[0,ii]\n", + "\n", + " plt.close()\n", + " con_plot = (wave_disp > 0)\n", + " plt.errorbar(wave_disp[con_plot], flux_disp[con_plot], yerr=err_disp[con_plot])\n", + " plt.ylim(-20, 100)\n", + " plt.show()\n", + "\n", + " # Wirte:\n", + " # Now make it into a Spectrum1D instance.\n", + " file_1d = '%sl3_nis_%s_%s_s%s_ndither%d_1d_opt.fits'%(DIR_OUT, filt, grism, id, ndither)\n", + "\n", + " if wave_disp[1] - wave_disp[0] < 0:\n", + " obs = Spectrum1D(spectral_axis=wave_disp[::-1]*u.um,\n", + " flux=flux_disp[::-1]*u.MJy,\n", + " uncertainty=StdDevUncertainty(err_disp[::-1]), unit='MJy')\n", + " else:\n", + " obs = Spectrum1D(spectral_axis=wave_disp*u.um,\n", + " flux=flux_disp*u.MJy,\n", + " uncertainty=StdDevUncertainty(err_disp), unit='MJy')\n", + "\n", + " obs.write(file_1d, format='tabular-fits', overwrite=True) " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From a44ed7fa059f207e20605329d64ef8ba13349277 Mon Sep 17 00:00:00 2001 From: Camilla Pacifici Date: Mon, 24 Jul 2023 15:24:30 -0400 Subject: [PATCH 02/62] Include notebook for catalog --- .../05_catalog_for_spectral_extraction.ipynb | 180 +++++++++++++++++- 1 file changed, 174 insertions(+), 6 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb b/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb index 85d1e7fa9..dbe922239 100644 --- a/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb @@ -18,6 +18,8 @@ "outputs": [], "source": [ "from astropy.table import Table\n", + "from astropy.coordinates import SkyCoord\n", + "from astropy import units as u\n", "from jdaviz import Imviz" ] }, @@ -39,7 +41,7 @@ "data_dir = '/Users/cpacifici/DATA/NGDEEP/image3/'\n", "file_f115w = data_dir + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv'\n", "file_f150w = data_dir + 'jw02079-o004_t001_niriss_clear-f150w_cat.ecsv'\n", - "file_f150w = data_dir + 'jw02079-o004_t001_niriss_clear-f200w_cat.ecsv'" + "file_f200w = data_dir + 'jw02079-o004_t001_niriss_clear-f200w_cat.ecsv'" ] }, { @@ -50,15 +52,52 @@ "outputs": [], "source": [ "cat_f115w = Table.read(file_f115w)\n", - "cat_f115w" + "cat_f150w = Table.read(file_f150w)\n", + "cat_f200w = Table.read(file_f200w)\n", + "print(len(cat_f150w))\n", + "cat_f150w['sky_centroid']" ] }, { "cell_type": "markdown", - "id": "b44f4b06", + "id": "fcb9e2cd", "metadata": {}, "source": [ - "## Filter interesting objects" + "### Plot in Imviz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53d84f30", + "metadata": {}, + "outputs": [], + "source": [ + "imagefile = data_dir + 'jw02079-o004_t001_niriss_clear-f150w_i2d.fits'\n", + "imviz = Imviz()\n", + "imviz.load_data(imagefile, data_label='F150W')\n", + "imviz.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a051cce9", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot markers for all objects\n", + "viewer = imviz.default_viewer\n", + "viewer.reset_markers()\n", + "t_f115w = Table({'coord': cat_f115w['sky_centroid']})\n", + "t_f150w = Table({'coord': cat_f150w['sky_centroid']})\n", + "t_f200w = Table({'coord': cat_f200w['sky_centroid']})\n", + "viewer.marker = {'color': 'green', 'alpha': 0.8, 'markersize': 30, 'fill': False}\n", + "viewer.add_markers(t_f115w, use_skycoord=True, marker_name='full_cat_f115w')\n", + "viewer.marker = {'color': 'orange', 'alpha': 0.8, 'markersize': 5, 'fill': True}\n", + "viewer.add_markers(t_f150w, use_skycoord=True, marker_name='full_cat_f150w')\n", + "viewer.marker = {'color': 'red', 'alpha': 0.8, 'markersize': 5, 'fill': True}\n", + "viewer.add_markers(t_f200w, use_skycoord=True, marker_name='full_cat_f200w')" ] }, { @@ -69,13 +108,142 @@ "## Rewrite catalogs with matching IDs" ] }, + { + "cell_type": "markdown", + "id": "efd1ffba", + "metadata": {}, + "source": [ + "### Match catalogs and find object in all three\n", + "\n", + "1. Match F115W to F150W -> f115w_base_matches\n", + "2. Match f115w_base_matches to F200W -> f115w_base_matches, f200w_try_matches\n", + "3. Rematch the matched F115W to F150W -> f115w_matches, f150w_try_matches" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b90adf2", + "metadata": {}, + "outputs": [], + "source": [ + "f115w_base = cat_f115w['sky_centroid']\n", + "f150w_try = cat_f150w['sky_centroid']\n", + "\n", + "max_sep = 1.0 * u.arcsec\n", + "idx, d2d, d3d = f115w_base.match_to_catalog_3d(f150w_try)\n", + "sep_constraint = d2d < max_sep\n", + "f115w_base_matches = cat_f115w[sep_constraint]\n", + "f150w_try_matches = cat_f150w[idx[sep_constraint]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ceeb065c", + "metadata": {}, + "outputs": [], + "source": [ + "f115w_base = f115w_base_matches['sky_centroid']\n", + "f200w_try = cat_f200w['sky_centroid']\n", + "\n", + "max_sep = 1.0 * u.arcsec\n", + "idx, d2d, d3d = f115w_base.match_to_catalog_3d(f200w_try)\n", + "sep_constraint = d2d < max_sep\n", + "f115w_base_matches = f115w_base_matches[sep_constraint]\n", + "f200w_try_matches = cat_f200w[idx[sep_constraint]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e7c7e7a", + "metadata": {}, + "outputs": [], + "source": [ + "f115w_base = f115w_base_matches['sky_centroid']\n", + "f150w_try = cat_f150w['sky_centroid']\n", + "\n", + "max_sep = 1.0 * u.arcsec\n", + "idx, d2d, d3d = f115w_base.match_to_catalog_3d(f150w_try)\n", + "sep_constraint = d2d < max_sep\n", + "f115w_matches = f115w_base_matches[sep_constraint]\n", + "f150w_try_matches = cat_f150w[idx[sep_constraint]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "977a5c32", + "metadata": {}, + "outputs": [], + "source": [ + "f115w_matches" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46bb3784", + "metadata": {}, + "outputs": [], + "source": [ + "# Check that they all have the same length\n", + "print(\"Length F115W match: \", len(f115w_matches))\n", + "print(\"Length F150W match: \", len(f150w_try_matches))\n", + "print(\"Length F200W match: \", len(f200w_try_matches))" + ] + }, + { + "cell_type": "markdown", + "id": "2f9085b9", + "metadata": {}, + "source": [ + "### Rewrite catalogs" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "9f54b03d", + "id": "4ffbe9ce", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "f115w_out_file = data_dir + 'jw02079-o004_t001_niriss_clear-f115w_cat_matched.ecsv'\n", + "f150w_out_file = data_dir + 'jw02079-o004_t001_niriss_clear-f150w_cat_matched.ecsv'\n", + "f200w_out_file = data_dir + 'jw02079-o004_t001_niriss_clear-f200w_cat_matched.ecsv'" + ] + }, + { + "cell_type": "markdown", + "id": "e7549208", + "metadata": {}, + "source": [ + "This is the critical step. Use the same `label` for the same targets in the three catalogs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3da16f16", + "metadata": {}, + "outputs": [], + "source": [ + "f150w_try_matches['label'] = f115w_matches['label']\n", + "f200w_try_matches['label'] = f115w_matches['label']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7bedf34d", + "metadata": {}, + "outputs": [], + "source": [ + "f115w_matches.write(f115w_out_file, format='ascii.ecsv', overwrite=True)\n", + "f150w_try_matches.write(f150w_out_file, format='ascii.ecsv', overwrite=True)\n", + "f200w_try_matches.write(f200w_out_file, format='ascii.ecsv', overwrite=True)" + ] } ], "metadata": { From 5cea903ad42f5b2978896c60f9525fb8153f62ac Mon Sep 17 00:00:00 2001 From: Camilla Pacifici Date: Mon, 24 Jul 2023 16:55:42 -0400 Subject: [PATCH 03/62] Improve style of notebook 00 --- .../00_mast_query_progID.ipynb | 59 ++++++++++++++++--- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb index dd2bca8c3..0ac48cb65 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb @@ -5,8 +5,33 @@ "id": "592da8da", "metadata": {}, "source": [ - "## Get observations from program ID\n", - "Inspired by the [JWebbinar session about MAST](https://github.com/spacetelescope/jwebbinar_prep/blob/main/mast_session/Crowded_Field/Crowded_Field.ipynb)." + "# Get observations from program ID\n", + "This notebook uses the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three NIRISS filters: F115W, F150W, and F200W.\n", + "\n", + "**Use case**: use MAST to download data products.
\n", + "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", + "**Tools**: astropy, astroquery, numpy
\n", + "**Cross-instrument**: all
\n", + "\n", + "**Content**\n", + "- [Imports](#imports)\n", + "- [Set up the query and the batches](#setup)\n", + "- [Filter and download products](#filter)\n", + "\n", + "\n", + "**Author**: Camilla Pacifici (cpacifici@stsci.edu)\n", + "**Last modified**: July 2023\n", + "\n", + "This notebook was inspired by the [JWebbinar session about MAST](https://github.com/spacetelescope/jwebbinar_prep/blob/main/mast_session/Crowded_Field/Crowded_Field.ipynb)." + ] + }, + { + "cell_type": "markdown", + "id": "ff93c82c", + "metadata": {}, + "source": [ + "\n", + "## Imports" ] }, { @@ -23,6 +48,15 @@ "import numpy as np" ] }, + { + "cell_type": "markdown", + "id": "008fcb1e", + "metadata": {}, + "source": [ + "\n", + "## Set up the query and the batches" + ] + }, { "cell_type": "markdown", "id": "9447551a", @@ -69,6 +103,15 @@ "print(\"How many batches?\", len(obs_batches))" ] }, + { + "cell_type": "markdown", + "id": "7e57c3c0", + "metadata": {}, + "source": [ + "\n", + "## Filter and download products" + ] + }, { "cell_type": "markdown", "id": "10c7d144", @@ -119,21 +162,19 @@ " dataproduct_type=[\"image\"],\n", " calib_level=[3]\n", " )\n", - " #filtered_products\n", " # Download products for these records.\n", " print('Products downloaded:')\n", " print(filtered_products['productFilename'])\n", - " #print(filtered_products)\n", " manifest = Observations.download_products(filtered_products)" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "de034fed", + "cell_type": "markdown", + "id": "8c0e1d71", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "\"Space" + ] } ], "metadata": { From b9b21245dce4787269d40d5989e4a00855ff5663 Mon Sep 17 00:00:00 2001 From: Camilla Pacifici Date: Mon, 24 Jul 2023 16:56:47 -0400 Subject: [PATCH 04/62] Minor improvements to notebook 05 --- .../05_catalog_for_spectral_extraction.ipynb | 78 ++++++++++++------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb b/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb index dbe922239..c5c1336e3 100644 --- a/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb @@ -7,7 +7,7 @@ "source": [ "# Create catalog for spectral extraction\n", "\n", - "Pick a few interesting objects and create catalogs for each filter with matching IDs for easy analysis after the extraction of the 2D spectra." + "Create catalogs for each filter with matching IDs for easy analysis after the extraction of the 2D spectra. No filtering is applied here. Make copy of original catalogs because they will be overwritten. If I overwrite the catalogs, I do not have to change all the association files." ] }, { @@ -17,6 +17,8 @@ "metadata": {}, "outputs": [], "source": [ + "import glob\n", + "import shutil\n", "from astropy.table import Table\n", "from astropy.coordinates import SkyCoord\n", "from astropy import units as u\n", @@ -60,7 +62,27 @@ }, { "cell_type": "markdown", - "id": "fcb9e2cd", + "id": "c9d738f3", + "metadata": {}, + "source": [ + "Make copy of the catalogs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d414e56b", + "metadata": {}, + "outputs": [], + "source": [ + "shutil.copy(file_f115w, data_dir+'jw02079-o004_t001_niriss_clear-f115w_cat_original.ecsv')\n", + "shutil.copy(file_f150w, data_dir+'jw02079-o004_t001_niriss_clear-f150w_cat_original.ecsv')\n", + "shutil.copy(file_f200w, data_dir+'jw02079-o004_t001_niriss_clear-f200w_cat_original.ecsv')" + ] + }, + { + "cell_type": "markdown", + "id": "81a83a9b", "metadata": {}, "source": [ "### Plot in Imviz" @@ -69,7 +91,7 @@ { "cell_type": "code", "execution_count": null, - "id": "53d84f30", + "id": "f50c9286", "metadata": {}, "outputs": [], "source": [ @@ -82,7 +104,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a051cce9", + "id": "47b4217a", "metadata": {}, "outputs": [], "source": [ @@ -110,7 +132,7 @@ }, { "cell_type": "markdown", - "id": "efd1ffba", + "id": "587b4376", "metadata": {}, "source": [ "### Match catalogs and find object in all three\n", @@ -123,7 +145,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7b90adf2", + "id": "4048f182", "metadata": {}, "outputs": [], "source": [ @@ -140,7 +162,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ceeb065c", + "id": "b8956cca", "metadata": {}, "outputs": [], "source": [ @@ -157,7 +179,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7e7c7e7a", + "id": "65915dfd", "metadata": {}, "outputs": [], "source": [ @@ -174,7 +196,7 @@ { "cell_type": "code", "execution_count": null, - "id": "977a5c32", + "id": "8e1da323", "metadata": {}, "outputs": [], "source": [ @@ -184,7 +206,7 @@ { "cell_type": "code", "execution_count": null, - "id": "46bb3784", + "id": "b2034693", "metadata": {}, "outputs": [], "source": [ @@ -196,36 +218,24 @@ }, { "cell_type": "markdown", - "id": "2f9085b9", + "id": "e36d9525", "metadata": {}, "source": [ "### Rewrite catalogs" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "4ffbe9ce", - "metadata": {}, - "outputs": [], - "source": [ - "f115w_out_file = data_dir + 'jw02079-o004_t001_niriss_clear-f115w_cat_matched.ecsv'\n", - "f150w_out_file = data_dir + 'jw02079-o004_t001_niriss_clear-f150w_cat_matched.ecsv'\n", - "f200w_out_file = data_dir + 'jw02079-o004_t001_niriss_clear-f200w_cat_matched.ecsv'" - ] - }, { "cell_type": "markdown", - "id": "e7549208", + "id": "516f62fa", "metadata": {}, "source": [ - "This is the critical step. Use the same `label` for the same targets in the three catalogs." + "Overwrite. This is the critical step. Use the same `label` for the same targets in the three catalogs." ] }, { "cell_type": "code", "execution_count": null, - "id": "3da16f16", + "id": "1ae2d0ab", "metadata": {}, "outputs": [], "source": [ @@ -236,14 +246,22 @@ { "cell_type": "code", "execution_count": null, - "id": "7bedf34d", + "id": "30362634", "metadata": {}, "outputs": [], "source": [ - "f115w_matches.write(f115w_out_file, format='ascii.ecsv', overwrite=True)\n", - "f150w_try_matches.write(f150w_out_file, format='ascii.ecsv', overwrite=True)\n", - "f200w_try_matches.write(f200w_out_file, format='ascii.ecsv', overwrite=True)" + "f115w_matches.write(file_f115w, format='ascii.ecsv', overwrite=True)\n", + "f150w_try_matches.write(file_f150w, format='ascii.ecsv', overwrite=True)\n", + "f200w_try_matches.write(file_f200w, format='ascii.ecsv', overwrite=True)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5eba8956", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 21c8656bd42893e0078dfbbd9c219fe72d539240 Mon Sep 17 00:00:00 2001 From: Camilla Pacifici Date: Tue, 25 Jul 2023 16:33:25 -0400 Subject: [PATCH 05/62] Finished notebooks 06 and 07 --- .../06_run_pipeline_spec2.ipynb | 363 ++++++++--- .../07_visualize_level2_mosviz.ipynb | 615 ++++++++++++++++++ ...t1d.ipynb => 08_specviz2d_extract1d.ipynb} | 0 ...ons.ipynb => 09_compare_extractions.ipynb} | 0 4 files changed, 892 insertions(+), 86 deletions(-) create mode 100644 notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb rename notebooks/NIRISS_WFSS_advanced/{07_specviz2d_extract1d.ipynb => 08_specviz2d_extract1d.ipynb} (100%) rename notebooks/NIRISS_WFSS_advanced/{08_compare_extractions.ipynb => 09_compare_extractions.ipynb} (100%) diff --git a/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb index a2f26e7c8..90bf30052 100644 --- a/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb @@ -5,7 +5,9 @@ "id": "6d674dc1", "metadata": {}, "source": [ - "# Run spec2 pipeline on rate files" + "# Run spec2 pipeline on rate files\n", + "\n", + "Need to copy all the needed files over to the working directory." ] }, { @@ -24,10 +26,9 @@ "outputs": [], "source": [ "# Packages that allow us to get information about objects:\n", - "import asdf\n", "import os\n", "# Update to a path in your system (see details below at \"Reference files\")\n", - "os.environ[\"CRDS_PATH\"] = \"~/crds_cache_may2023/\"\n", + "os.environ[\"CRDS_PATH\"] = \"/Users/cpacifici/crds_cache_july2023/\"\n", "os.environ[\"CRDS_SERVER_URL\"] = \"https://jwst-crds.stsci.edu\"\n" ] }, @@ -40,40 +41,19 @@ "source": [ "# The calwebb_spec and spec3 pipelines\n", "from jwst.pipeline import Spec2Pipeline\n", - "from jwst.pipeline import Spec3Pipeline\n", - "\n", - "# individual steps\n", - "from jwst.assign_wcs import AssignWcsStep\n", - "from jwst.assign_wcs import nirspec\n", - "from jwst.background import BackgroundStep\n", - "from jwst.imprint import ImprintStep\n", - "from jwst.msaflagopen import MSAFlagOpenStep\n", - "from jwst.extract_2d import Extract2dStep\n", - "from jwst.srctype import SourceTypeStep\n", - "from jwst.wavecorr import WavecorrStep\n", - "from jwst.flatfield import FlatFieldStep\n", - "from jwst.pathloss import PathLossStep\n", - "from jwst.photom import PhotomStep\n", - "from jwst.cube_build import CubeBuildStep\n", - "from jwst.extract_1d import Extract1dStep\n", - "from jwst.combine_1d import Combine1dStep\n", "\n", "# data models\n", "from jwst import datamodels\n", "\n", - "# associations\n", - "from jwst.associations import asn_from_list\n", - "from jwst.associations.lib.rules_level3_base import DMS_Level3_Base\n", - "\n", - "# astropy\n", - "from astropy.io import fits\n", - "from astropy.utils.data import download_file\n", - "import astropy.units as u\n", - "from astropy import wcs\n", - "from astropy.wcs import WCS\n", - "from astropy.visualization import ImageNormalize, ManualInterval, LogStretch, LinearStretch, AsinhStretch\n", + "# utility\n", + "from astropy.table import Table\n", + "import numpy as np\n", + "import json\n", + "import glob\n", + "import shutil\n", "\n", - "from matplotlib import pyplot as plt" + "# jdaviz\n", + "from jdaviz import Mosviz" ] }, { @@ -87,12 +67,54 @@ "print('jwst',jwst.__version__)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "168c2146", + "metadata": {}, + "outputs": [], + "source": [ + "# Update appropriately\n", + "data_dir_in = \"/Users/cpacifici/DATA/NGDEEP/rate/\"\n", + "data_dir_images = \"/Users/cpacifici/DATA/NGDEEP/image3/\"\n", + "data_dir_asn_level2 = \"/Users/cpacifici/DATA/NGDEEP/asn_level2/\"\n", + "data_dir_out_spec2 = \"/Users/cpacifici/DATA/NGDEEP/spec2/\"" + ] + }, + { + "cell_type": "markdown", + "id": "1aa37d93", + "metadata": {}, + "source": [ + "## Read list of rate files for imaging" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b5572ab", + "metadata": {}, + "outputs": [], + "source": [ + "listrate_file = './list_ngdeep_rate.csv'\n", + "listrate = Table.read(listrate_file)\n", + "listrate[np.where((listrate['FILTER']=='GR150R') | (listrate['FILTER']=='GR150C'))]" + ] + }, + { + "cell_type": "markdown", + "id": "133f26bb", + "metadata": {}, + "source": [ + "## Run spec2 on all GR150R/C images using the provided association files" + ] + }, { "cell_type": "markdown", - "id": "67a46e39", + "id": "18cd35b0", "metadata": {}, "source": [ - "## Files" + "### Check an association file for level 2" ] }, { @@ -102,14 +124,78 @@ "metadata": {}, "outputs": [], "source": [ - "association_level2 = './jw02079-o004_f115w_f150wcat_asn.json'\n", - "rate = './jw02079004001_05101_00001_nis_rate.fits'\n", - "\n", - "#association_level2 = './jw02079-o004_f150w_f150wcat_asn.json'\n", - "#rate = './jw02079004002_11101_00001_nis_rate.fits'\n", - "\n", - "#association_level2 = './jw02079-o004_f200w_f150wcat_asn.json'\n", - "#rate = './jw02079004003_05101_00001_nis_rate.fits'\n" + "asnfile = data_dir_asn_level2 + 'jw02079-o004_20230622t175524_spec2_00001_asn.json'\n", + "asn_data = json.load(open(asnfile))\n", + "print(asn_data['products'][0]['members'][0]['expname'])\n", + "asn_data" + ] + }, + { + "cell_type": "markdown", + "id": "026e94a9", + "metadata": {}, + "source": [ + "### Copy all needed files to the current directory" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de5d11ec", + "metadata": {}, + "outputs": [], + "source": [ + "asn_spec2 = glob.glob(data_dir_asn_level2+'*spec2*')\n", + "for file in asn_spec2:\n", + " shutil.copy(file, './')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd56b572", + "metadata": {}, + "outputs": [], + "source": [ + "rate_clear = listrate[np.where((listrate['FILTER']=='GR150R') | (listrate['FILTER']=='GR150C'))]['FILENAME']\n", + "for file in rate_clear:\n", + " shutil.copy(data_dir_in+file, './')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d765ef2c", + "metadata": {}, + "outputs": [], + "source": [ + "catalogfiles = glob.glob(data_dir_images+'*cat.ecsv')\n", + "for file in catalogfiles:\n", + " shutil.copy(file, './')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2fa2925", + "metadata": {}, + "outputs": [], + "source": [ + "imagefiles = glob.glob(data_dir_images+'*i2d.fits')\n", + "for file in imagefiles:\n", + " shutil.copy(file, './')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "531a997b", + "metadata": {}, + "outputs": [], + "source": [ + "segfiles = glob.glob(data_dir_images+'*segm.fits')\n", + "for file in segfiles:\n", + " shutil.copy(file, './')" ] }, { @@ -120,110 +206,215 @@ "## Run spec2" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "b114db48", + "metadata": {}, + "outputs": [], + "source": [ + "asn_spec2_local = glob.glob('./jw02079-o004_*_spec2_*_asn.json')\n", + "asn_spec2_local" + ] + }, { "cell_type": "code", "execution_count": null, "id": "b2d5b653", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "for file in asn_spec2_local:\n", + " print(file)\n", + " spec2 = Spec2Pipeline() \n", + " result = spec2.call(file, \n", + " steps={'bkg_subtract':{'skip':False},\n", + " 'extract_2d':{'wfss_nbright':500,\n", + " 'wfss_extract_half_height': 25,\n", + " },\n", + " 'photom':{'skip':False},\n", + " },\n", + " save_results=True,\n", + " output_dir=data_dir_out_spec2)\n" + ] + }, + { + "cell_type": "markdown", + "id": "b06abd18", + "metadata": {}, + "source": [ + "### Delete files not needed anymore" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8c142f2", "metadata": {}, "outputs": [], "source": [ - "output_dir = './pipeline_output/'\n", - "# Create an instance of the pipeline class\n", - "spec2 = Spec2Pipeline()\n", + "dir_to_be_del = './files_to_be_deleted'\n", + "if not os.path.exists(dir_to_be_del):\n", + " os.mkdir(dir_to_be_del)\n", + " \n", + "for file in asn_spec2_local:\n", + " shutil.move(file, dir_to_be_del)\n", + " \n", + "for file in glob.glob('./*rate.fits'):\n", + " shutil.move(file, dir_to_be_del)\n", "\n", - "# Set some parameters that pertain to the entire pipeline\n", - "#spec2.save_results = True\n", - "#spec2.output_dir = output_dir\n", + "for file in glob.glob('./*niriss_clear*'):\n", + " shutil.move(file, dir_to_be_del)\n", "\n", - "# Here you could some parameters that pertain to some of the individual steps\n", - "#spec2.extract_2d.mmag_extract = 20 # Minimum magnitude of objects to extract for WFSS data\n", - "#spec2.extract_1d.bkg_fit = 'poly' # Fit a polynomial to the background values for each column or row\n", - "#spec2.bkg_subtract.skip = True # Skip the default image-from-image background subtraction method\n", - "\n", - "# Call the call() method, using an input association file.\n", - "# Call method reinitialized the default values, so parameters need to be changed inside the call\n", - "result = spec2.call(association_level2, \n", - " steps={'bkg_subtract':{'skip':True},\n", - " 'extract_2d':{'wfss_nbright':500,\n", - " 'wfss_extract_half_height': 25,\n", - " },\n", - " 'photom':{'skip':True},\n", - " },\n", - " save_results=True,\n", - " output_dir=output_dir)\n" + "shutil.rmtree(dir_to_be_del)" + ] + }, + { + "cell_type": "markdown", + "id": "b0ef4f23", + "metadata": {}, + "source": [ + "## Visualize output in Mosviz\n", + "I will use a different instance of Mosviz per filter." + ] + }, + { + "cell_type": "markdown", + "id": "4c0abc3c", + "metadata": {}, + "source": [ + "### F115W GR150R" ] }, { "cell_type": "code", "execution_count": null, - "id": "41661aec", + "id": "46fe0a2d", "metadata": {}, "outputs": [], "source": [ - "calfile = './pipeline_output/jw02079004001_05101_00001_nis_cal.fits'\n", - "calimage = fits.open(calfile)" + "directimage = data_dir_images + 'jw02079-o004_t001_niriss_clear-f115w_i2d.fits'\n", + "catalogfile = data_dir_images + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv'\n", + "calfile = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_cal.fits'\n", + "x1dfile = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_x1d.fits'" ] }, { "cell_type": "code", "execution_count": null, - "id": "b1eb9efd", + "id": "f8cd0041", "metadata": {}, "outputs": [], "source": [ - "source = 4\n", - "print(calimage['SCI',source].header['SOURCEID'])" + "dir_mosviz = './mosviz_stuff'\n", + "if not os.path.exists(dir_mosviz):\n", + " os.mkdir(dir_mosviz)\n", + "\n", + "shutil.copy(directimage, dir_mosviz)\n", + "shutil.copy(catalogfile, dir_mosviz)\n", + "shutil.copy(calfile, dir_mosviz)\n", + "shutil.copy(x1dfile, dir_mosviz)" ] }, { "cell_type": "code", "execution_count": null, - "id": "46fe0a2d", + "id": "287a5570", "metadata": {}, "outputs": [], "source": [ - "data = calimage['SCI',source].data\n", - "plt.imshow(data, origin='lower', vmin=0.4, vmax=0.8)\n", - "plt.colorbar()" + "mosviz = Mosviz()\n", + "mosviz.load_data(directory=dir_mosviz, instrument='niriss')\n", + "mosviz.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36b5c6c7", + "metadata": {}, + "outputs": [], + "source": [ + "plotopt = mosviz.plugins['Plot Options']\n", + "plotopt.viewer = 'image-viewer'\n", + "plotopt.stretch_vmin = 0.15\n", + "plotopt.stretch_vmax = 0.4\n", + "\n", + "plotopt.viewer = 'spectrum-2d-viewer'\n", + "plotopt.stretch_vmin = 0\n", + "plotopt.stretch_vmax = 2" ] }, { "cell_type": "markdown", - "id": "197cfb65", + "id": "7be6e906", "metadata": {}, "source": [ - "There is no background subtraction so this is absolutely useless." + "We add a column to the table to take notes and mark the interesting objects with emission lines." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3179068", + "metadata": {}, + "outputs": [], + "source": [ + "mosviz.add_column('Notes')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2cca12b", + "metadata": {}, + "outputs": [], + "source": [ + "# Filter F115W grism GR150R\n", + "mosviz.update_column('Notes', 'line', row=23) # ID 216\n", + "mosviz.update_column('Notes', 'line maybe', row=25) # ID 243\n", + "mosviz.update_column('Notes', 'line maybe', row=27) # ID 188\n", + "mosviz.update_column('Notes', 'line', row=35) # ID 250\n", + "mosviz.update_column('Notes', 'line', row=53) # ID 21\n", + "mosviz.update_column('Notes', 'line', row=56) # ID 100\n", + "mosviz.update_column('Notes', 'line', row=77) # ID 23\n", + "mosviz.update_column('Notes', 'line', row=85) # ID 71\n", + "mosviz.update_column('Notes', 'line', row=86) # ID 179\n", + "mosviz.update_column('Notes', 'line', row=99) # ID 245\n", + "mosviz.update_column('Notes', 'line maybe', row=100) # ID 109\n", + "mosviz.update_column('Notes', 'line', row=117) # ID 22\n", + "mosviz.update_column('Notes', 'line', row=134) # ID 2\n", + "mosviz.update_column('Notes', 'line', row=137) # ID 4\n", + "mosviz.update_column('Notes', 'line maybe', row=157) # ID 113\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "f7c09858", + "id": "1e87d45f", "metadata": {}, "outputs": [], "source": [ - "x1dfile = './pipeline_output/jw02079004002_11101_00001_nis_x1d.fits'\n", - "x1dspec = fits.open(x1dfile)" + "table_f115w_grr = mosviz.to_table()" ] }, { "cell_type": "code", "execution_count": null, - "id": "930ac35b", + "id": "eee938db", "metadata": {}, "outputs": [], "source": [ - "data = x1dspec['EXTRACT1D',source].data\n", - "plt.plot(data['WAVELENGTH'], data['FLUX'])\n", - "plt.xlabel('wavelength (um)')\n", - "plt.ylabel('flux (adu)')\n", - "plt.show()" + "# Save table to file\n", + "table_f115w_grr.write('./f115w_gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" ] }, { "cell_type": "code", "execution_count": null, - "id": "353bad8c", + "id": "64a3bba8", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb b/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb new file mode 100644 index 000000000..32fa33c4d --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb @@ -0,0 +1,615 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a7b71e63", + "metadata": {}, + "source": [ + "# Visualize level2 output in Mosviz\n", + "I will use a different instance of Mosviz per filter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fab6cc54", + "metadata": {}, + "outputs": [], + "source": [ + "# utility\n", + "import os\n", + "from astropy.table import Table, join\n", + "import numpy as np\n", + "import glob\n", + "import shutil\n", + "\n", + "# jdaviz\n", + "from jdaviz import Mosviz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a24e3f9", + "metadata": {}, + "outputs": [], + "source": [ + "# Update appropriately\n", + "data_dir_images = \"/Users/cpacifici/DATA/NGDEEP/image3/\"\n", + "data_dir_out_spec2 = \"/Users/cpacifici/DATA/NGDEEP/spec2/\"" + ] + }, + { + "cell_type": "markdown", + "id": "067cf0c4", + "metadata": {}, + "source": [ + "## F115W GR150R" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b00e9d9d", + "metadata": {}, + "outputs": [], + "source": [ + "directimage = data_dir_images + 'jw02079-o004_t001_niriss_clear-f115w_i2d.fits'\n", + "catalogfile = data_dir_images + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv'\n", + "calfile = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_cal.fits'\n", + "x1dfile = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_x1d.fits'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b016350e", + "metadata": {}, + "outputs": [], + "source": [ + "dir_mosviz = './mosviz_stuff'\n", + "if os.path.exists(dir_mosviz):\n", + " shutil.rmtree(dir_mosviz)\n", + "\n", + "os.mkdir(dir_mosviz)\n", + "\n", + "shutil.copy(directimage, dir_mosviz)\n", + "shutil.copy(catalogfile, dir_mosviz)\n", + "shutil.copy(calfile, dir_mosviz)\n", + "shutil.copy(x1dfile, dir_mosviz)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b7f1cc0", + "metadata": {}, + "outputs": [], + "source": [ + "mosviz1 = Mosviz()\n", + "mosviz1.load_data(directory=dir_mosviz, instrument='niriss')\n", + "mosviz1.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "128aec4b", + "metadata": {}, + "outputs": [], + "source": [ + "plotopt = mosviz1.plugins['Plot Options']\n", + "plotopt.viewer = 'image-viewer'\n", + "plotopt.stretch_vmin = 0.15\n", + "plotopt.stretch_vmax = 0.4\n", + "\n", + "plotopt.viewer = 'spectrum-2d-viewer'\n", + "plotopt.stretch_vmin = 0\n", + "plotopt.stretch_vmax = 2" + ] + }, + { + "cell_type": "markdown", + "id": "3a13745c", + "metadata": {}, + "source": [ + "We add a column to the table to take notes and mark the interesting objects with emission lines." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31703246", + "metadata": {}, + "outputs": [], + "source": [ + "mosviz1.add_column('Notes')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eed32ee9", + "metadata": {}, + "outputs": [], + "source": [ + "# Filter F115W grism GR150R\n", + "mosviz1.update_column('Notes', 'line', row=23) # ID 216\n", + "mosviz1.update_column('Notes', 'line maybe', row=25) # ID 243\n", + "mosviz1.update_column('Notes', 'line maybe', row=27) # ID 188\n", + "mosviz1.update_column('Notes', 'line', row=35) # ID 250\n", + "mosviz1.update_column('Notes', 'line', row=53) # ID 21\n", + "mosviz1.update_column('Notes', 'line', row=56) # ID 100\n", + "mosviz1.update_column('Notes', 'line', row=77) # ID 23\n", + "mosviz1.update_column('Notes', 'line', row=85) # ID 71\n", + "mosviz1.update_column('Notes', 'line', row=86) # ID 179\n", + "mosviz1.update_column('Notes', 'line', row=99) # ID 245\n", + "mosviz1.update_column('Notes', 'line maybe', row=100) # ID 109\n", + "mosviz1.update_column('Notes', 'line', row=117) # ID 22\n", + "mosviz1.update_column('Notes', 'line', row=134) # ID 2\n", + "mosviz1.update_column('Notes', 'line', row=137) # ID 4\n", + "mosviz1.update_column('Notes', 'line maybe', row=157) # ID 113\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3807fd8", + "metadata": {}, + "outputs": [], + "source": [ + "table_f115w_grr = mosviz1.to_table()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49115f40", + "metadata": {}, + "outputs": [], + "source": [ + "# Save table to file\n", + "table_f115w_grr.write('./f115w_gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" + ] + }, + { + "cell_type": "markdown", + "id": "765e6081", + "metadata": {}, + "source": [ + "## F150W GR150R" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6cfbcda7", + "metadata": {}, + "outputs": [], + "source": [ + "directimage = data_dir_images + 'jw02079-o004_t001_niriss_clear-f150w_i2d.fits'\n", + "catalogfile = data_dir_images + 'jw02079-o004_t001_niriss_clear-f150w_cat.ecsv'\n", + "calfile = data_dir_out_spec2 + 'jw02079004002_09101_00001_nis_cal.fits'\n", + "x1dfile = data_dir_out_spec2 + 'jw02079004002_09101_00001_nis_x1d.fits'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d920b143", + "metadata": {}, + "outputs": [], + "source": [ + "dir_mosviz = './mosviz_stuff'\n", + "if os.path.exists(dir_mosviz):\n", + " shutil.rmtree(dir_mosviz)\n", + "\n", + "os.mkdir(dir_mosviz)\n", + "\n", + "shutil.copy(directimage, dir_mosviz)\n", + "shutil.copy(catalogfile, dir_mosviz)\n", + "shutil.copy(calfile, dir_mosviz)\n", + "shutil.copy(x1dfile, dir_mosviz)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f6ca5b1", + "metadata": {}, + "outputs": [], + "source": [ + "mosviz2 = Mosviz()\n", + "mosviz2.load_data(directory=dir_mosviz, instrument='niriss')\n", + "mosviz2.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99fb216e", + "metadata": {}, + "outputs": [], + "source": [ + "plotopt = mosviz2.plugins['Plot Options']\n", + "plotopt.viewer = 'image-viewer'\n", + "plotopt.stretch_vmin = 0.15\n", + "plotopt.stretch_vmax = 0.5\n", + "\n", + "plotopt.viewer = 'spectrum-2d-viewer'\n", + "plotopt.stretch_vmin = 0\n", + "plotopt.stretch_vmax = 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3aac083", + "metadata": {}, + "outputs": [], + "source": [ + "mosviz2.add_column('Notes')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5409284c", + "metadata": {}, + "outputs": [], + "source": [ + "# Filter F150W grism GR150R\n", + "mosviz2.update_column('Notes', 'line', row=14) # ID 170\n", + "mosviz2.update_column('Notes', 'line', row=21) # ID 150\n", + "mosviz2.update_column('Notes', 'maybe line', row=34) # ID 38\n", + "mosviz2.update_column('Notes', 'line', row=36) # ID 195\n", + "mosviz2.update_column('Notes', 'maybe line', row=44) # ID 184\n", + "mosviz2.update_column('Notes', 'maybe line', row=47) # ID 213\n", + "mosviz2.update_column('Notes', 'maybe line', row=57) # ID 25\n", + "mosviz2.update_column('Notes', 'line', row=60) # ID 218\n", + "mosviz2.update_column('Notes', 'line', row=62) # ID 21\n", + "mosviz2.update_column('Notes', 'line', row=63) # ID 37\n", + "mosviz2.update_column('Notes', 'maybe line', row=66) # ID 145\n", + "mosviz2.update_column('Notes', 'maybe line', row=70) # ID 19\n", + "mosviz2.update_column('Notes', 'line', row=71) # ID 207\n", + "mosviz2.update_column('Notes', 'line', row=73) # ID 15\n", + "mosviz2.update_column('Notes', 'line', row=74) # ID 63\n", + "mosviz2.update_column('Notes', 'line', row=77) # ID 109 # promising!\n", + "mosviz2.update_column('Notes', 'maybe line', row=81) # ID 23\n", + "mosviz2.update_column('Notes', 'maybe line', row=83) # ID 54\n", + "mosviz2.update_column('Notes', 'maybe line', row=84) # ID 190\n", + "mosviz2.update_column('Notes', 'maybe line', row=100) # ID 114\n", + "mosviz2.update_column('Notes', 'line', row=106) # ID 193 \n", + "mosviz2.update_column('Notes', 'line', row=110) # ID 135 \n", + "mosviz2.update_column('Notes', 'line', row=128) # ID 132 \n", + "mosviz2.update_column('Notes', 'maybe line', row=130) # ID 149\n", + "mosviz2.update_column('Notes', 'line', row=138) # ID 7 \n", + "mosviz2.update_column('Notes', 'line', row=139) # ID 44 \n", + "mosviz2.update_column('Notes', 'line', row=142) # ID 2 \n", + "mosviz2.update_column('Notes', 'maybe line', row=157) # ID 16\n", + "mosviz2.update_column('Notes', 'line', row=159) # ID 28 \n", + "mosviz2.update_column('Notes', 'line', row=161) # ID 41 \n", + "mosviz2.update_column('Notes', 'line', row=162) # ID 93 \n", + "mosviz2.update_column('Notes', 'maybe line', row=176) # ID 11\n", + "mosviz2.update_column('Notes', 'maybe line', row=177) # ID 113\n", + "mosviz2.update_column('Notes', 'line', row=180) # ID 27 \n", + "mosviz2.update_column('Notes', 'line', row=194) # ID 86 \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d03b024a", + "metadata": {}, + "outputs": [], + "source": [ + "table_f150w_grr = mosviz2.to_table()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2acaebe5", + "metadata": {}, + "outputs": [], + "source": [ + "# Save table to file\n", + "table_f150w_grr.write('./f150w_gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" + ] + }, + { + "cell_type": "markdown", + "id": "0fd0bc12", + "metadata": {}, + "source": [ + "## F200W GR150R" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce46b974", + "metadata": {}, + "outputs": [], + "source": [ + "directimage = data_dir_images + 'jw02079-o004_t001_niriss_clear-f200w_i2d.fits'\n", + "catalogfile = data_dir_images + 'jw02079-o004_t001_niriss_clear-f200w_cat.ecsv'\n", + "calfile = data_dir_out_spec2 + 'jw02079004003_03101_00001_nis_cal.fits'\n", + "x1dfile = data_dir_out_spec2 + 'jw02079004003_03101_00001_nis_x1d.fits'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "849aad09", + "metadata": {}, + "outputs": [], + "source": [ + "dir_mosviz = './mosviz_stuff'\n", + "if os.path.exists(dir_mosviz):\n", + " shutil.rmtree(dir_mosviz)\n", + "\n", + "os.mkdir(dir_mosviz)\n", + "\n", + "shutil.copy(directimage, dir_mosviz)\n", + "shutil.copy(catalogfile, dir_mosviz)\n", + "shutil.copy(calfile, dir_mosviz)\n", + "shutil.copy(x1dfile, dir_mosviz)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1716821d", + "metadata": {}, + "outputs": [], + "source": [ + "mosviz3 = Mosviz()\n", + "mosviz3.load_data(directory=dir_mosviz, instrument='niriss')\n", + "mosviz3.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4314b1f4", + "metadata": {}, + "outputs": [], + "source": [ + "plotopt = mosviz3.plugins['Plot Options']\n", + "plotopt.viewer = 'image-viewer'\n", + "plotopt.stretch_vmin = 0.15\n", + "plotopt.stretch_vmax = 0.5\n", + "\n", + "plotopt.viewer = 'spectrum-2d-viewer'\n", + "plotopt.stretch_vmin = 0\n", + "plotopt.stretch_vmax = 4" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b086e2ce", + "metadata": {}, + "outputs": [], + "source": [ + "mosviz3.add_column('Notes')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "821fd610", + "metadata": {}, + "outputs": [], + "source": [ + "# Filter F200W grism GR150R\n", + "mosviz3.update_column('Notes', 'maybe line', row=25) # ID 216\n", + "mosviz3.update_column('Notes', 'line', row=39) # ID 123 # base key\n", + "mosviz3.update_column('Notes', 'line', row=48) # ID 92 \n", + "mosviz3.update_column('Notes', 'line', row=53) # ID 146 \n", + "mosviz3.update_column('Notes', 'maybe line', row=57) # ID 116\n", + "mosviz3.update_column('Notes', 'line', row=65) # ID 37\n", + "mosviz3.update_column('Notes', 'line', row=73) # ID 155\n", + "mosviz3.update_column('Notes', 'line', row=81) # ID 192\n", + "mosviz3.update_column('Notes', 'line', row=97) # ID 109\n", + "mosviz3.update_column('Notes', 'line', row=102) # ID 201\n", + "mosviz3.update_column('Notes', 'line', row=107) # ID 149\n", + "mosviz3.update_column('Notes', 'maybe line', row=114) # ID 98\n", + "mosviz3.update_column('Notes', 'line', row=116) # ID 52\n", + "mosviz3.update_column('Notes', 'maybe line', row=121) # ID 169\n", + "mosviz3.update_column('Notes', 'line', row=129) # ID 32\n", + "mosviz3.update_column('Notes', 'line', row=135) # ID 16\n", + "mosviz3.update_column('Notes', 'maybe line', row=139) # ID 136\n", + "mosviz3.update_column('Notes', 'line', row=165) # ID 120\n", + "mosviz3.update_column('Notes', 'line', row=173) # ID 41\n", + "mosviz3.update_column('Notes', 'line', row=188) # ID 93\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae6b5fa0", + "metadata": {}, + "outputs": [], + "source": [ + "table_f200w_grr = mosviz3.to_table()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c47b96a7", + "metadata": {}, + "outputs": [], + "source": [ + "# Save table to file\n", + "table_f200w_grr.write('./f200w_gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" + ] + }, + { + "cell_type": "markdown", + "id": "f9fe8290", + "metadata": {}, + "source": [ + "### Delete files not needed anymore" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d68b90f", + "metadata": {}, + "outputs": [], + "source": [ + "if os.path.exists(dir_mosviz):\n", + " shutil.rmtree(dir_mosviz)" + ] + }, + { + "cell_type": "markdown", + "id": "6e42598e", + "metadata": {}, + "source": [ + "## Combine catalogs and flags" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eafd7e06", + "metadata": {}, + "outputs": [], + "source": [ + "table_f115w_grr.sort(['Identifier'])\n", + "table_f150w_grr.sort(['Identifier'])\n", + "table_f200w_grr.sort(['Identifier'])" + ] + }, + { + "cell_type": "markdown", + "id": "c2d812b3", + "metadata": {}, + "source": [ + "### Remove non-useful columns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "68576c23", + "metadata": {}, + "outputs": [], + "source": [ + "table_f115w_grr.remove_columns(['Table Index', '1D Spectra', '2D Spectra', 'Images', 'Filter/Grating', 'Redshift'])\n", + "table_f150w_grr.remove_columns(['Table Index', '1D Spectra', '2D Spectra', 'Images', 'Filter/Grating', 'Redshift'])\n", + "table_f200w_grr.remove_columns(['Table Index', '1D Spectra', '2D Spectra', 'Images', 'Filter/Grating', 'Redshift'])" + ] + }, + { + "cell_type": "markdown", + "id": "37ebf01c", + "metadata": {}, + "source": [ + "### Rename notes column" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd14d9ba", + "metadata": {}, + "outputs": [], + "source": [ + "table_f115w_grr.rename_column('Notes', 'Notes_F115W')\n", + "table_f150w_grr.rename_column('Notes', 'Notes_F150W')\n", + "table_f200w_grr.rename_column('Notes', 'Notes_F200W')" + ] + }, + { + "cell_type": "markdown", + "id": "d43b7202", + "metadata": {}, + "source": [ + "### Join tables" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3e2c095", + "metadata": {}, + "outputs": [], + "source": [ + "table_flags = join(table_f115w_grr, table_f150w_grr, keys='Identifier')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce4f93b5", + "metadata": {}, + "outputs": [], + "source": [ + "table_flags = join(table_flags, table_f200w_grr, keys='Identifier')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65c4f85e", + "metadata": {}, + "outputs": [], + "source": [ + "table_flags" + ] + }, + { + "cell_type": "markdown", + "id": "47cb9a52", + "metadata": {}, + "source": [ + "### Write out flag file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ff34311", + "metadata": {}, + "outputs": [], + "source": [ + "table_flags.write('./gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c02ecbd0", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/NIRISS_WFSS_advanced/07_specviz2d_extract1d.ipynb b/notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/07_specviz2d_extract1d.ipynb rename to notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/08_compare_extractions.ipynb b/notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/08_compare_extractions.ipynb rename to notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb From e3464bf1695999908ebc887ec59bcef92a75b258 Mon Sep 17 00:00:00 2001 From: Camilla Pacifici Date: Tue, 1 Aug 2023 15:50:52 -0400 Subject: [PATCH 06/62] Continued with notebooks 08 and 09 --- .../01_check_files_headers_progID.ipynb | 4 +- .../03_imviz_level2.ipynb | 4 +- .../04_run_pipeline_imaging_level2and3.ipynb | 8 +- .../06_run_pipeline_spec2.ipynb | 146 +--------- .../07_visualize_level2_mosviz.ipynb | 5 +- .../08_specviz2d_extract1d.ipynb | 267 ++++++++++++++++-- .../09_compare_extractions.ipynb | 126 +++++++++ 7 files changed, 387 insertions(+), 173 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb index 6a1ec04c3..ca2d96474 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb @@ -5,7 +5,9 @@ "id": "continental-machinery", "metadata": {}, "source": [ - "# Create list of files from MAST download" + "# Create list of files from MAST download\n", + "\n", + "Credit Jo Taylor." ] }, { diff --git a/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb b/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb index d07713bc0..fafa999b8 100644 --- a/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb @@ -5,7 +5,9 @@ "id": "28608b44", "metadata": {}, "source": [ - "# Check flat fielded images from NGDEEP" + "# Check flat fielded images from NGDEEP\n", + "\n", + "This is to look at the images and see if there are problems." ] }, { diff --git a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb index ecfc7fe18..0ffb50fb6 100644 --- a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb @@ -5,7 +5,9 @@ "id": "7a935d7c", "metadata": {}, "source": [ - "# Run pipeline and create catalog of sources" + "# Run pipeline and create catalog of sources\n", + "\n", + "This runs the imaging pipeline. Credit JWebbinar notebooks." ] }, { @@ -123,7 +125,7 @@ "id": "0421c67b", "metadata": {}, "source": [ - "### Move all needed files to the current directory" + "### Copy all needed files to the current directory" ] }, { @@ -317,7 +319,7 @@ " 'deblend':True,\n", " },\n", " 'tweakreg':{'snr_threshold':20,\n", - " 'abs_refcat':'GAIADR2',\n", + " 'abs_refcat':'GAIADR2', #Try DR3\n", " 'save_catalogs':True,\n", " 'searchrad':3.0,\n", " 'kernel_fwhm':2.302,\n", diff --git a/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb index 90bf30052..d61832bfb 100644 --- a/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb @@ -7,7 +7,11 @@ "source": [ "# Run spec2 pipeline on rate files\n", "\n", - "Need to copy all the needed files over to the working directory." + "Need to copy all the needed files over to the working directory.\n", + "\n", + "**Author notes**:\n", + "- Anything I need to know on the spec2 steps?\n", + "- I have to try turning on the correction for contamination" ] }, { @@ -271,146 +275,6 @@ "shutil.rmtree(dir_to_be_del)" ] }, - { - "cell_type": "markdown", - "id": "b0ef4f23", - "metadata": {}, - "source": [ - "## Visualize output in Mosviz\n", - "I will use a different instance of Mosviz per filter." - ] - }, - { - "cell_type": "markdown", - "id": "4c0abc3c", - "metadata": {}, - "source": [ - "### F115W GR150R" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "46fe0a2d", - "metadata": {}, - "outputs": [], - "source": [ - "directimage = data_dir_images + 'jw02079-o004_t001_niriss_clear-f115w_i2d.fits'\n", - "catalogfile = data_dir_images + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv'\n", - "calfile = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_cal.fits'\n", - "x1dfile = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_x1d.fits'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f8cd0041", - "metadata": {}, - "outputs": [], - "source": [ - "dir_mosviz = './mosviz_stuff'\n", - "if not os.path.exists(dir_mosviz):\n", - " os.mkdir(dir_mosviz)\n", - "\n", - "shutil.copy(directimage, dir_mosviz)\n", - "shutil.copy(catalogfile, dir_mosviz)\n", - "shutil.copy(calfile, dir_mosviz)\n", - "shutil.copy(x1dfile, dir_mosviz)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "287a5570", - "metadata": {}, - "outputs": [], - "source": [ - "mosviz = Mosviz()\n", - "mosviz.load_data(directory=dir_mosviz, instrument='niriss')\n", - "mosviz.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36b5c6c7", - "metadata": {}, - "outputs": [], - "source": [ - "plotopt = mosviz.plugins['Plot Options']\n", - "plotopt.viewer = 'image-viewer'\n", - "plotopt.stretch_vmin = 0.15\n", - "plotopt.stretch_vmax = 0.4\n", - "\n", - "plotopt.viewer = 'spectrum-2d-viewer'\n", - "plotopt.stretch_vmin = 0\n", - "plotopt.stretch_vmax = 2" - ] - }, - { - "cell_type": "markdown", - "id": "7be6e906", - "metadata": {}, - "source": [ - "We add a column to the table to take notes and mark the interesting objects with emission lines." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b3179068", - "metadata": {}, - "outputs": [], - "source": [ - "mosviz.add_column('Notes')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d2cca12b", - "metadata": {}, - "outputs": [], - "source": [ - "# Filter F115W grism GR150R\n", - "mosviz.update_column('Notes', 'line', row=23) # ID 216\n", - "mosviz.update_column('Notes', 'line maybe', row=25) # ID 243\n", - "mosviz.update_column('Notes', 'line maybe', row=27) # ID 188\n", - "mosviz.update_column('Notes', 'line', row=35) # ID 250\n", - "mosviz.update_column('Notes', 'line', row=53) # ID 21\n", - "mosviz.update_column('Notes', 'line', row=56) # ID 100\n", - "mosviz.update_column('Notes', 'line', row=77) # ID 23\n", - "mosviz.update_column('Notes', 'line', row=85) # ID 71\n", - "mosviz.update_column('Notes', 'line', row=86) # ID 179\n", - "mosviz.update_column('Notes', 'line', row=99) # ID 245\n", - "mosviz.update_column('Notes', 'line maybe', row=100) # ID 109\n", - "mosviz.update_column('Notes', 'line', row=117) # ID 22\n", - "mosviz.update_column('Notes', 'line', row=134) # ID 2\n", - "mosviz.update_column('Notes', 'line', row=137) # ID 4\n", - "mosviz.update_column('Notes', 'line maybe', row=157) # ID 113\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1e87d45f", - "metadata": {}, - "outputs": [], - "source": [ - "table_f115w_grr = mosviz.to_table()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eee938db", - "metadata": {}, - "outputs": [], - "source": [ - "# Save table to file\n", - "table_f115w_grr.write('./f115w_gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb b/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb index 32fa33c4d..103261593 100644 --- a/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb @@ -6,7 +6,10 @@ "metadata": {}, "source": [ "# Visualize level2 output in Mosviz\n", - "I will use a different instance of Mosviz per filter." + "I will use a different instance of Mosviz per filter.\n", + "\n", + "**Author note**:\n", + "- worth including GR150C too?" ] }, { diff --git a/notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb b/notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb index 3f7c330e6..0deef5547 100644 --- a/notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb @@ -1,5 +1,28 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "08d94d39", + "metadata": {}, + "source": [ + "# Spectral extraction" + ] + }, + { + "cell_type": "markdown", + "id": "e88b5d78", + "metadata": {}, + "source": [ + "- select the object(s) with lines in all grisms\n", + "- extract the three spectra with specreduce\n", + "\n", + "**Author notes**:\n", + "- Combine all the exposures for a single target\n", + "- Find something more automatic on multiple spectra using specreduce?\n", + "- Extract with boxcar and optimal (also self profile)\n", + "- Still need to take care of NaNs in specreduce\n" + ] + }, { "cell_type": "code", "execution_count": null, @@ -7,45 +30,120 @@ "metadata": {}, "outputs": [], "source": [ - "from jdaviz import Specviz2d, Specviz\n", - "from specutils import Spectrum1D\n", + "# utility\n", + "import os\n", + "import glob\n", + "import shutil\n", "import numpy as np\n", + "\n", + "# astropy\n", + "from astropy.table import Table, join\n", "from astropy.io import fits\n", - "from matplotlib import pyplot as plt\n", - "from astropy import units as u" + "from astropy import units as u\n", + "\n", + "# jdaviz\n", + "from jdaviz import Specviz2d, Specviz\n", + "\n", + "# specutils\n", + "from specutils import Spectrum1D\n", + "\n", + "# \n", + "from matplotlib import pyplot as plt\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "b7d71a35", + "id": "1d04f1bb", "metadata": {}, "outputs": [], "source": [ - "calfile = './runpipeline_spec2/pipeline_output/jw02079004001_05101_00001_nis_cal.fits'\n", - "calimage = fits.open(calfile)" + "# Update appropriately\n", + "data_dir_images = \"/Users/cpacifici/DATA/NGDEEP/image3/\"\n", + "data_dir_out_spec2 = \"/Users/cpacifici/DATA/NGDEEP/spec2/\"" + ] + }, + { + "cell_type": "markdown", + "id": "833b183f", + "metadata": {}, + "source": [ + "## Select files and interesting object" ] }, { "cell_type": "code", "execution_count": null, - "id": "486ae51a", + "id": "5b199602", "metadata": {}, "outputs": [], "source": [ - "for i in range(1,15):\n", - " print(i, calimage['SCI',i].header['SOURCEID'])" + "calfile_f115w = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_cal.fits'\n", + "calfile_f150w = data_dir_out_spec2 + 'jw02079004002_09101_00001_nis_cal.fits'\n", + "calfile_f200w = data_dir_out_spec2 + 'jw02079004003_03101_00001_nis_cal.fits'\n", + "calfiles = [calfile_f115w, calfile_f150w, calfile_f200w]" ] }, { "cell_type": "code", "execution_count": null, - "id": "f6526475", + "id": "660447e0", "metadata": {}, "outputs": [], "source": [ - "source = 3\n", - "print(calimage['SCI',source].header['SOURCEID'])" + "# My favorite object from the exploration in notebook 07\n", + "object_id = 109" + ] + }, + { + "cell_type": "markdown", + "id": "7caba000", + "metadata": {}, + "source": [ + "### Find correct extension in the files" + ] + }, + { + "cell_type": "markdown", + "id": "ec7b8cac", + "metadata": {}, + "source": [ + "**Developer note:** there might be a way to know how many SCI extensions there are in a fits file, but I do not know how. For now I hard code `201` because this is the number of objects in the catalog that was used to generate the `cal` files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e6895ee", + "metadata": {}, + "outputs": [], + "source": [ + "ext_num = []\n", + "for file in calfiles:\n", + " with fits.open(file) as hdu:\n", + " print(file)\n", + " for ii in range(1, 202):\n", + " if hdu['SCI',ii].header['SOURCEID']==object_id:\n", + " ext_num.append(ii)\n", + "ext_num" + ] + }, + { + "cell_type": "markdown", + "id": "8daf2d8f", + "metadata": {}, + "source": [ + "## Load in Specviz2d and extract 1D spectrum\n", + "\n", + "**Developer note**: the NaNs are not yet treated properly by Specviz2d (and specreduce). Need to mask them to zeros before proceeding." + ] + }, + { + "cell_type": "markdown", + "id": "9273c73b", + "metadata": {}, + "source": [ + "### F115W" ] }, { @@ -55,11 +153,12 @@ "metadata": {}, "outputs": [], "source": [ - "data = calimage['SCI',source].data\n", + "filter_num = 0\n", + "calimage = fits.open(calfiles[filter_num])\n", + "data = calimage['SCI',ext_num[filter_num]].data\n", + "data = np.transpose(data)\n", "data = np.nan_to_num(data)\n", - "invertdata = np.flip(data, 1)\n", - "wave = calimage['WAVELENGTH',source].data[0]\n", - "invertwave = np.flip(wave)" + "wave = calimage['WAVELENGTH',ext_num[filter_num]].data[:,0]" ] }, { @@ -69,7 +168,7 @@ "metadata": {}, "outputs": [], "source": [ - "spec2d = Spectrum1D(flux=invertdata*u.Jy, spectral_axis=invertwave*u.um)" + "spec2d = Spectrum1D(flux=data*u.Jy, spectral_axis=wave*u.um)" ] }, { @@ -91,37 +190,125 @@ "metadata": {}, "outputs": [], "source": [ - "spec1d_pix = specviz2d.get_data('Spectrum 1D')" + "spec1d_f115w = specviz2d.get_data('Spectrum 1D')" + ] + }, + { + "cell_type": "markdown", + "id": "03aa7058", + "metadata": {}, + "source": [ + "### F150W" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a20e31d1", + "metadata": {}, + "outputs": [], + "source": [ + "filter_num = 1\n", + "calimage = fits.open(calfiles[filter_num])\n", + "data = calimage['SCI',ext_num[filter_num]].data\n", + "data = np.transpose(data)\n", + "data = np.nan_to_num(data)\n", + "wave = calimage['WAVELENGTH',ext_num[filter_num]].data[:,0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3919e94", + "metadata": {}, + "outputs": [], + "source": [ + "spec2d = Spectrum1D(flux=data*u.Jy, spectral_axis=wave*u.um)" ] }, { "cell_type": "code", "execution_count": null, - "id": "154924cb", + "id": "1ee3d806", "metadata": {}, "outputs": [], "source": [ - "spec1d_f115w = Spectrum1D(flux=spec1d_pix.flux, spectral_axis=invertwave*u.um)" + "specviz2d2 = Specviz2d()\n", + "specviz2d2.load_data(spec2d)\n", + "specviz2d2.show()" ] }, { "cell_type": "code", "execution_count": null, - "id": "20718648", + "id": "9e161b0e", "metadata": {}, "outputs": [], "source": [ - "spec1d_f150w = Spectrum1D(flux=spec1d_pix.flux, spectral_axis=invertwave*u.um)" + "spec1d_f150w = specviz2d2.get_data('Spectrum 1D')" + ] + }, + { + "cell_type": "markdown", + "id": "71a9814f", + "metadata": {}, + "source": [ + "### F200W" ] }, { "cell_type": "code", "execution_count": null, - "id": "2477a732", + "id": "ea2e0c45", "metadata": {}, "outputs": [], "source": [ - "spec1d_f200w = Spectrum1D(flux=spec1d_pix.flux, spectral_axis=invertwave*u.um)" + "filter_num = 2\n", + "calimage = fits.open(calfiles[filter_num])\n", + "data = calimage['SCI',ext_num[filter_num]].data\n", + "data = np.transpose(data)\n", + "data = np.nan_to_num(data)\n", + "wave = calimage['WAVELENGTH',ext_num[filter_num]].data[:,0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de1815d0", + "metadata": {}, + "outputs": [], + "source": [ + "spec2d = Spectrum1D(flux=data*u.Jy, spectral_axis=wave*u.um)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b2a6e89", + "metadata": {}, + "outputs": [], + "source": [ + "specviz2d3 = Specviz2d()\n", + "specviz2d3.load_data(spec2d)\n", + "specviz2d3.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "613b1402", + "metadata": {}, + "outputs": [], + "source": [ + "spec1d_f200w = specviz2d3.get_data('Spectrum 1D')" + ] + }, + { + "cell_type": "markdown", + "id": "5ce78c0f", + "metadata": {}, + "source": [ + "## Open in Specviz" ] }, { @@ -138,10 +325,38 @@ "specviz.show()" ] }, + { + "cell_type": "markdown", + "id": "e974d11f", + "metadata": {}, + "source": [ + "Load a line list and measure the redshift of the target (spoiler: z=2.23)" + ] + }, + { + "cell_type": "markdown", + "id": "684e262b", + "metadata": {}, + "source": [ + "### Save spectra as fits files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "695c3d5c", + "metadata": {}, + "outputs": [], + "source": [ + "spec1d_f115w.write('source109_f115w.fits')\n", + "spec1d_f150w.write('source109_f150w.fits')\n", + "spec1d_f200w.write('source109_f200w.fits')" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "0645e33f", + "id": "8b59d88b", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb b/notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb index e0cc02bb9..f5afd17a5 100755 --- a/notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb @@ -1,5 +1,131 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare custom extraction and pipeline extraction\n", + "\n", + "**Author note**:\n", + "- Need to run spec3?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# utility\n", + "import os\n", + "import glob\n", + "import shutil\n", + "import numpy as np\n", + "\n", + "# astropy\n", + "from astropy.table import Table, join\n", + "from astropy.io import fits\n", + "from astropy import units as u\n", + "\n", + "# specutils\n", + "from specutils import Spectrum1D\n", + "\n", + "# matplotlib\n", + "from matplotlib import pyplot as plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Update appropriately\n", + "data_dir_out_spec2 = \"/Users/cpacifici/DATA/NGDEEP/spec2/\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "x1dfile_f115w = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_x1d.fits'\n", + "x1dfile_f150w = data_dir_out_spec2 + 'jw02079004002_09101_00001_nis_x1d.fits'\n", + "x1dfile_f200w = data_dir_out_spec2 + 'jw02079004003_03101_00001_nis_x1d.fits'\n", + "x1dfiles = [x1dfile_f115w, x1dfile_f150w, x1dfile_f200w]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# My favorite object from the exploration in notebook 07\n", + "object_id = 109" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ext_num = []\n", + "spec1d = []\n", + "for file in x1dfiles:\n", + " with fits.open(file) as hdu:\n", + " print(file)\n", + " for ii in range(1, 202):\n", + " if hdu['EXTRACT1D',ii].header['SOURCEID']==object_id:\n", + " ext_num.append(ii)\n", + " spec1d.append(hdu['EXTRACT1D',ii].data)\n", + "ext_num" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "custom_ext_f115w = fits.open('source109_f115w.fits')\n", + "custom_ext_f150w = fits.open('source109_f150w.fits')\n", + "custom_ext_f200w = fits.open('source109_f200w.fits')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(16,10))\n", + "plt.plot(spec1d[0]['WAVELENGTH'], spec1d[0]['FLUX'], '-', color='blue')\n", + "plt.plot(custom_ext_f115w[1].data['wavelength'], custom_ext_f115w[1].data['flux']/10000000, '--', color='blue')\n", + "plt.plot(spec1d[1]['WAVELENGTH'], spec1d[1]['FLUX'], '-', color='green')\n", + "plt.plot(custom_ext_f150w[1].data['wavelength'], custom_ext_f150w[1].data['flux']/10000000, '--', color='green')\n", + "plt.plot(spec1d[2]['WAVELENGTH'], spec1d[2]['FLUX'], '-', color='red')\n", + "plt.plot(custom_ext_f200w[1].data['wavelength'], custom_ext_f200w[1].data['flux']/10000000, '--', color='red')\n", + "plt.ylim(0,0.00002)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Not finished...still working on the rest" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, From d556a511bc42f40a2bc31d71597bfd69fd8818d8 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 16 Oct 2023 13:10:57 -0400 Subject: [PATCH 07/62] Updates to mast download for NIRISS. Adding comments & new sections --- .../00_mast_query_progID.ipynb | 226 +++++++++++++++--- 1 file changed, 199 insertions(+), 27 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb index 0ac48cb65..17698675b 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb @@ -6,11 +6,11 @@ "metadata": {}, "source": [ "# Get observations from program ID\n", - "This notebook uses the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three NIRISS filters: F115W, F150W, and F200W.\n", + "This notebook uses the python [astroquery.mast Observations](https://astroquery.readthedocs.io/en/latest/mast/mast_obsquery.html) class of the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three NIRISS filters: F115W, F150W, and F200W using both GR150R and GR150C grisms.\n", "\n", "**Use case**: use MAST to download data products.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: astropy, astroquery, numpy
\n", + "**Tools**: astropy, astroquery, numpy, (yaml)
\n", "**Cross-instrument**: all
\n", "\n", "**Content**\n", @@ -19,8 +19,8 @@ "- [Filter and download products](#filter)\n", "\n", "\n", - "**Author**: Camilla Pacifici (cpacifici@stsci.edu)\n", - "**Last modified**: July 2023\n", + "**Author**: Camilla Pacifici (cpacifici@stsci.edu) & Rachel Plesha (rplesha@stsci.edu)\n", + "**Last modified**: October 2023\n", "\n", "This notebook was inspired by the [JWebbinar session about MAST](https://github.com/spacetelescope/jwebbinar_prep/blob/main/mast_session/Crowded_Field/Crowded_Field.ipynb)." ] @@ -57,50 +57,89 @@ "## Set up the query and the batches" ] }, + { + "cell_type": "markdown", + "id": "9562f354", + "metadata": {}, + "source": [ + "The observations class in ``astroquery.mast`` is used to download JWST data. Use the metadata function to see the available search options and their descriptions.\n", + "\n", + "Note that for JWST, the instrument names have a specific format. More information about that can be found at: https://outerspace.stsci.edu/display/MASTDOCS/JWST+Instrument+Names " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3185204", + "metadata": {}, + "outputs": [], + "source": [ + "Observations.get_metadata(\"observations\")" + ] + }, { "cell_type": "markdown", "id": "9447551a", "metadata": {}, "source": [ - "Select the proposal ID, instrument, and some useful keywords (filters in this case)." + "The two most common ways to download specific datasets are by using the [proposal ID](https://www.stsci.edu/jwst/science-execution/program-information) or by using the [observation ID](https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/file_naming.html)." + ] + }, + { + "cell_type": "markdown", + "id": "8cedd24c", + "metadata": {}, + "source": [ + "#### Search with Proposal ID" ] }, { "cell_type": "code", "execution_count": null, "id": "569efa6d", - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ + "# Select the proposal ID, instrument, and some useful keywords (filters in this case).\n", "obs_table = Observations.query_criteria(obs_collection=[\"JWST\"], \n", " instrument_name=[\"NIRISS/IMAGE\", \"NIRISS/WFSS\"],\n", " provenance_name=[\"CALJWST\"], # Executed observations\n", " filters=['F115W','F150W','F200W'],\n", " proposal_id=[2079],\n", " )\n", + "\n", + "print(len(obs_table), 'files found')\n", + "# look at what was obtained in this query\n", "obs_table" ] }, { "cell_type": "markdown", - "id": "9daaeaf9", + "id": "8cf89e82", "metadata": {}, "source": [ - "I could list all the products, but if there are too many it could time out. Let us devide the observations in baches and then filter and download one batch at a time." + "#### Search with Observation ID\n", + "The observation ID (obs_id) allows for flexibility of searching by the proposal ID and the observation ID because of how the JWST filenames are structured. More information about the JWST file naming conventions can be found at: https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/file_naming.html.\n", + "\n", + "Additionally, there is flexibility using wildcards inside of the search criteria. For example, instead of specifying both \"NIRISS/IMAGE\" and \"NIRISS/WFSS\", we can specify \"NIRISS*\", which picks up both file modes. The wildcard also works within the obs_id, so we do not have to list all of the different IDs." ] }, { "cell_type": "code", "execution_count": null, - "id": "b8a653a9", + "id": "6ddce5a5", "metadata": {}, "outputs": [], "source": [ - "batch_size = 5\n", - "# Let's split up ``obs_table`` into batches according to our batch size.\n", - "obs_batches = [obs_table[i:i+batch_size] for i in range(0, len(obs_table), batch_size)]\n", - "single_group = obs_batches[0] # Useful to inspect the files obtained from one group\n", - "print(\"How many batches?\", len(obs_batches))" + "# Obtain a list to download from a specific list of observation IDs instead\n", + "obs_id_table = Observations.query_criteria(instrument_name=[\"NIRISS*\"],\n", + " provenance_name=[\"CALJWST\"], # Executed observations\n", + " obs_id=['jw02079-o004*'], # Searching for PID 2079 observation 004\n", + " ) \n", + "\n", + "print(len(obs_id_table), 'files found')" ] }, { @@ -112,12 +151,44 @@ "## Filter and download products" ] }, + { + "cell_type": "markdown", + "id": "9daaeaf9", + "metadata": {}, + "source": [ + "If there are too many files to download, the API will time out. Instead, it is better to divide the observations in batches to download one at a time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8a653a9", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "batch_size = 5 # 5 files at a time maximizes the download speed.\n", + "\n", + "# Let's split up our list of files, ``obs_table``, into batches according to our batch size.\n", + "obs_batches = [obs_table[i:i+batch_size] for i in range(0, len(obs_table), batch_size)]\n", + "print(\"How many batches?\", len(obs_batches))\n", + "\n", + "single_group = obs_batches[0] # Useful to inspect the files obtained from one group\n", + "print(\"Inspect the first batch:\")\n", + "single_group" + ] + }, { "cell_type": "markdown", "id": "10c7d144", "metadata": {}, "source": [ - "Select the type of products needed. I do one at a time. I need:\n", + "Select the type of products needed. The various levels are:\n", + "- uncalibrated files\n", + " - productType=[\"SCIENCE\"]\n", + " - productSubGroupDescription=['UNCAL']\n", + " - calib_level=[1]\n", "- rate images\n", " - productType=[\"SCIENCE\"]\n", " - productSubGroupDescription=['RATE']\n", @@ -133,16 +204,112 @@ " - calib_level=[3]" ] }, + { + "cell_type": "markdown", + "id": "61ec7fc6", + "metadata": {}, + "source": [ + "#### Filtering" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "616ff15c", + "id": "fd82b142", "metadata": {}, "outputs": [], + "source": [ + "# creating a dictionary of the above information to use for inspection of the filtering function\n", + "file_dict = {'uncal' : {'product_type' : 'SCIENCE',\n", + " 'productSubGroupDescription' : 'UNCAL',\n", + " 'calib_level' : [1]},\n", + " 'rate' : {'product_type' : 'SCIENCE',\n", + " 'productSubGroupDescription' : 'RATE',\n", + " 'calib_level' : [2]},\n", + " 'level2_association' : {'product_type' : 'INFO',\n", + " 'productSubGroupDescription' : 'ASN',\n", + " 'calib_level' : [2]},\n", + " 'level3_association' : {'product_type' : 'INFO',\n", + " 'productSubGroupDescription' : 'ASN',\n", + " 'calib_level' : [3]},\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fed7ca65", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "## Look at the files existing for each of these different levels\n", + "files_to_download = []\n", + "for index, batch_exposure in enumerate(single_group):\n", + " \n", + " print('*'*50)\n", + " print(f\"Exposure #{index+1} ({batch_exposure['obs_id']})\")\n", + " # pull out the product names from the list to filter\n", + " products = Observations.get_product_list(batch_exposure)\n", + " \n", + " for filetype, query_dict in file_dict.items():\n", + " print('File type:', filetype)\n", + " filtered_products = Observations.filter_products(products,\n", + " productType=query_dict['product_type'],\n", + " productSubGroupDescription=query_dict['productSubGroupDescription'],\n", + " calib_level=query_dict['calib_level'],\n", + " )\n", + " print(filtered_products['productFilename'])\n", + " files_to_download.extend(filtered_products['productFilename'])\n", + " print()\n", + " print('*'*50)\n" + ] + }, + { + "cell_type": "markdown", + "id": "e98933e6", + "metadata": {}, + "source": [ + "We can see that for each exposure in the observation list, there are many files associated that need to be downloaded. This is why we need to work in batches to download.\n", + "\n", + "\n", + "#### Downloading\n", + "To actually download the products, provide ``Observations.download_products()`` with a list of the filtered products. \n", + "\n", + "If the data are proprietary, you may also need to set up your API token. **NEVER** commit your token to a public repository. An alternative is to create a separate configuration file (config_file.yaml) that is readable only to you and has a key: 'mast_token' : _API token_\n", + "\n", + "To make create a new API token visit to following link: \n", + " https://auth.mast.stsci.edu/token?suggested_name=Astroquery&suggested_scope=mast:exclusive_access" + ] + }, + { + "cell_type": "raw", + "id": "fc1897a3", + "metadata": {}, + "source": [ + "# if needed, replace Observations.download_products() with the following:\n", + "\n", + "import yaml\n", + "\n", + "with open(config_file) as f:\n", + " mast_config = yaml.safe_load(f)\n", + " \n", + "mysession = Observations(mast_config['mast_token'])\n", + "mysession.download_products(filtered_products)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "616ff15c", + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "# Now let's get the products for each batch of observations, and filter down to only the products of interest.\n", "for index, batch in enumerate(obs_batches):\n", - "#for index, batch in enumerate(single_group): # Useful to inspect the files obtained from one batch\n", " \n", " # Progress indicator...\n", " print('\\n'+f'Batch #{index+1}')\n", @@ -150,22 +317,27 @@ " # Make a list of the `obsid` identifiers from our Astropy table of observations to get products for.\n", " obsids = batch['obsid']\n", " print('Working with the following ``obsid``s:')\n", - " print(list(obsids))\n", + " for number, obs_text in zip(obsids, batch['obs_id']):\n", + " print(f\"{number} : {obs_text}\")\n", " \n", " # Get list of products \n", " products = Observations.get_product_list(obsids)\n", " \n", - " # Filter the products to only get science products of calibration level 3 \n", + " # Filter the products to only get only the products of interest\n", " filtered_products = Observations.filter_products(products, \n", - " productType=[\"INFO\"],\n", - " productSubGroupDescription=['ASN'],\n", - " dataproduct_type=[\"image\"],\n", - " calib_level=[3]\n", + " productType=[\"SCIENCE\", \"INFO\"],\n", + " productSubGroupDescription=[\"UNCAL\", \"RATE\", \"ASN\"],\n", + " calib_level=[1, 2, 3]\n", " )\n", " # Download products for these records.\n", - " print('Products downloaded:')\n", - " print(filtered_products['productFilename'])\n", - " manifest = Observations.download_products(filtered_products)" + " manifest = Observations.download_products(filtered_products,\n", + " download_dir='data',\n", + " flat=True, # astroquery v0.4.7 or later only\n", + " ) \n", + " #print('Products downloaded:\\n', filtered_products['productFilename'])\n", + " \n", + " # only downloading the first batch of 5 observations\n", + " break # comment this out if you want to download everything" ] }, { @@ -193,7 +365,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.9.12" } }, "nbformat": 4, From 5b498e0498a9ef6456e906fb18b61dc541393ce9 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 16 Oct 2023 13:23:21 -0400 Subject: [PATCH 08/62] Removing uncal files from the download call --- .../NIRISS_WFSS_advanced/00_mast_query_progID.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb index 17698675b..149a76e90 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb @@ -87,7 +87,7 @@ }, { "cell_type": "markdown", - "id": "8cedd24c", + "id": "ded88209", "metadata": {}, "source": [ "#### Search with Proposal ID" @@ -117,7 +117,7 @@ }, { "cell_type": "markdown", - "id": "8cf89e82", + "id": "28b20df5", "metadata": {}, "source": [ "#### Search with Observation ID\n", @@ -206,7 +206,7 @@ }, { "cell_type": "markdown", - "id": "61ec7fc6", + "id": "2109ecff", "metadata": {}, "source": [ "#### Filtering" @@ -326,8 +326,8 @@ " # Filter the products to only get only the products of interest\n", " filtered_products = Observations.filter_products(products, \n", " productType=[\"SCIENCE\", \"INFO\"],\n", - " productSubGroupDescription=[\"UNCAL\", \"RATE\", \"ASN\"],\n", - " calib_level=[1, 2, 3]\n", + " productSubGroupDescription=[\"RATE\", \"ASN\"], # Not using \"UNCAL\" here since we can start with the rate files\n", + " calib_level=[2, 3] # not using 1 here since not getting the UNCAL files\n", " )\n", " # Download products for these records.\n", " manifest = Observations.download_products(filtered_products,\n", From 550132f728189f943a4137526e23358036ad28b4 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 16 Oct 2023 13:30:55 -0400 Subject: [PATCH 09/62] A few comments & moving the files with python --- .../01_check_files_headers_progID.ipynb | 170 +++++++++++++++++- 1 file changed, 161 insertions(+), 9 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb index ca2d96474..7671e49cf 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb @@ -12,14 +12,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "nasty-belize", "metadata": {}, "outputs": [], "source": [ "from astropy.io import fits\n", "import pandas as pd\n", - "import glob" + "import glob\n", + "import os" ] }, { @@ -32,18 +33,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "3db7c79f", "metadata": { "scrolled": false }, "outputs": [], "source": [ - "df = pd.DataFrame(columns=[\"FILENAME\", \"TARG_RA\", \"TARG_DEC\", \"FILTER\", \"PUPIL\", \"PATT_NUM\", \"NUMDTHPT\", \"XOFFSET\", \"YOFFSET\"])\n", + "# set up an empty dataframe initially to fill in\n", + "df = pd.DataFrame(columns=[\"FILENAME\",\n", + " \"TARG_RA\", \n", + " \"TARG_DEC\", \n", + " \"FILTER\", # Grism; GR150R/GR150C\n", + " \"PUPIL\", # Filter used; F090W, F115W, F140M, F150W F158M, F200W\n", + " \"PATT_NUM\", # Position number within dither pattern for WFSS\n", + " \"NUMDTHPT\", # Total number of points in entire dither pattern\n", + " \"XOFFSET\", # X offset from pattern starting position for NIRISS (arcsec)\n", + " \"YOFFSET\"]) # Y offset from pattern starting position for NIRISS (arcsec)\n", "\n", - "for file in glob.glob(\"./mastDownload/JWST/jw02079004*/*rate.fits\"):\n", - " image = fits.open(file)\n", + "for ratefile in glob.glob(\"data/*rate.fits\"):\n", + " image = fits.open(ratefile)\n", " \n", + " # create a new dataframe for each file\n", " df2 = pd.DataFrame({\"FILENAME\" : [image[0].header[\"FILENAME\"]],\n", " \"TARG_RA\" : [image[0].header[\"TARG_RA\"]],\n", " \"TARG_DEC\" : [image[0].header[\"TARG_DEC\"]],\n", @@ -55,6 +66,7 @@ " \"YOFFSET\" : [image[0].header[\"YOFFSET\"]]\n", " })\n", " \n", + " # merge the two dataframes together to create a dataframe with all \n", " df = pd.concat([df, df2], ignore_index = True, axis = 0)\n", " \n", "dfsort = df.sort_values('FILENAME', ignore_index = True)" @@ -70,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "a77d81da", "metadata": {}, "outputs": [], @@ -99,8 +111,148 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "6bc1b5c4", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Moved: data/jw02079004001_04101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004001_04101_00002_nis_rate.fits\n", + "Moved: data/jw02079004003_06101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004003_06101_00002_nis_rate.fits\n", + "Moved: data/jw02079004002_10101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_10101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_06101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_06101_00003_nis_rate.fits\n", + "Moved: data/jw02079004003_02101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004003_02101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_10101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_10101_00002_nis_rate.fits\n", + "Moved: data/jw02079004001_04101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_04101_00001_nis_rate.fits\n", + "Moved: data/jw02079004001_12101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004001_12101_00003_nis_rate.fits\n", + "Moved: data/jw02079004003_06101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004003_06101_00001_nis_rate.fits\n", + "Moved: data/jw02079004003_04101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004003_04101_00002_nis_rate.fits\n", + "Moved: data/jw02079004001_10101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004001_10101_00004_nis_rate.fits\n", + "Moved: data/jw02079004001_06101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004001_06101_00002_nis_rate.fits\n", + "Moved: data/jw02079004002_04101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_04101_00003_nis_rate.fits\n", + "Moved: data/jw02079004001_02101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_02101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_11101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_11101_00002_nis_rate.fits\n", + "Moved: data/jw02079004002_12101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_12101_00001_nis_rate.fits\n", + "Moved: data/jw02079004001_08101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_08101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_11101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_11101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_04101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004002_04101_00004_nis_rate.fits\n", + "Moved: data/jw02079004002_12101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_12101_00002_nis_rate.fits\n", + "Moved: data/jw02079004001_10101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004001_10101_00003_nis_rate.fits\n", + "Moved: data/jw02079004003_04101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004003_04101_00001_nis_rate.fits\n", + "Moved: data/jw02079004001_06101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_06101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_12101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_12101_00003_nis_rate.fits\n", + "Moved: data/jw02079004002_04101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_04101_00001_nis_rate.fits\n", + "Moved: data/jw02079004003_04101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004003_04101_00004_nis_rate.fits\n", + "Moved: data/jw02079004001_10101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004001_10101_00002_nis_rate.fits\n", + "Moved: data/jw02079004001_06101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004001_06101_00003_nis_rate.fits\n", + "Moved: data/jw02079004003_04101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004003_04101_00003_nis_rate.fits\n", + "Moved: data/jw02079004001_10101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_10101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_11101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_11101_00003_nis_rate.fits\n", + "Moved: data/jw02079004002_04101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_04101_00002_nis_rate.fits\n", + "Moved: data/jw02079004002_06101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_06101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_10101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_10101_00003_nis_rate.fits\n", + "Moved: data/jw02079004001_04101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004001_04101_00004_nis_rate.fits\n", + "Moved: data/jw02079004001_12101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004001_12101_00002_nis_rate.fits\n", + "Moved: data/jw02079004003_06101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004003_06101_00003_nis_rate.fits\n", + "Moved: data/jw02079004002_02101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_02101_00001_nis_rate.fits\n", + "Moved: data/jw02079004002_08101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_08101_00001_nis_rate.fits\n", + "Moved: data/jw02079004001_12101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_12101_00001_nis_rate.fits\n", + "Moved: data/jw02079004001_04101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004001_04101_00003_nis_rate.fits\n", + "Moved: data/jw02079004002_10101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004002_10101_00004_nis_rate.fits\n", + "Moved: data/jw02079004002_06101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_06101_00002_nis_rate.fits\n", + "Unrecognized index: spec2 or asn\n", + "Moved: data/jw02079-o004_20231008t143836_image3_00005_asn.json to data/NGDEEP/asn_level3/jw02079-o004_20231008t143836_image3_00005_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00623_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00623_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00444_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00444_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00740_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00740_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00550_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00550_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00551_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00551_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00031_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00031_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00698_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00698_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00699_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00699_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00329_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00329_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00328_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00328_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00106_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00106_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00107_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00107_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00148_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00148_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00296_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00296_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image3_00009_asn.json to data/NGDEEP/asn_level3/jw02079-o004_20231008t143836_image3_00009_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00476_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00476_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00477_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00477_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00401_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00401_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00400_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00400_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image3_00002_asn.json to data/NGDEEP/asn_level3/jw02079-o004_20231008t143836_image3_00002_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00624_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00624_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00625_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00625_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00255_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00255_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00033_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00033_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00032_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00032_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00254_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00254_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00105_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00105_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00104_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00104_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00179_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00179_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00180_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00180_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00181_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00181_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00549_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00549_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00548_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00548_asn.json\n", + "Unrecognized index: spec2 or asn\n", + "Unrecognized index: spec2 or asn\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00592_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00592_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00475_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00475_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00402_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00402_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00403_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00403_asn.json\n", + "Unrecognized index: spec3 or asn\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00252_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00252_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00253_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00253_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00696_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00696_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00697_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00697_asn.json\n", + "Moved: data/jw02079-o004_20231008t143836_image2_00327_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00327_asn.json\n" + ] + } + ], + "source": [ + "# first, make all of the new directories\n", + "topdir = 'data/NGDEEP'\n", + "if not os.path.exists(topdir):\n", + " os.mkdir(topdir)\n", + " print('Created:', topdir)\n", + "\n", + "for subdir in ['rate', 'asn_level2', 'asn_level3']:\n", + " new_subdir = os.path.join(topdir, subdir)\n", + " if not os.path.exists(new_subdir):\n", + " os.mkdir(new_subdir)\n", + " print('Created:', new_subdir)\n", + "\n", + "file_dict = {'rate' : 'rate',\n", + " 'image3' : 'asn_level3',\n", + " 'image2' : 'asn_level2'}\n", + "\n", + "# now move all of the files to the appropriate locations\n", + "for filename in glob.glob('data/*.fits') + glob.glob('data/*.json'):\n", + " try:\n", + " index = filename.split('_')[2] # json files; looking for image2/image3\n", + " subdir = file_dict[index]\n", + " except KeyError:\n", + " try:\n", + " index2 = filename.split('_')[-1].split('.')[0] # rate files\n", + " subdir = file_dict[index2]\n", + " except KeyError:\n", + " print(f'Unrecognized index: {index} or {index2}')\n", + " continue\n", + " \n", + " new_file = os.path.join(topdir, subdir, os.path.basename(filename))\n", + " os.rename(filename, new_file)\n", + " print(f'Moved: {filename} to {new_file}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "935d05d9", "metadata": {}, "outputs": [], "source": [] @@ -122,7 +274,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.9.12" } }, "nbformat": 4, From 07ee0efe0d3cf37fa7791293f5a273b1371841ca Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 16 Oct 2023 14:45:49 -0400 Subject: [PATCH 10/62] Merging notebook 01 into notebook 02 instead --- .../00_mast_query_progID.ipynb | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb index 149a76e90..c6793817a 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb @@ -10,7 +10,7 @@ "\n", "**Use case**: use MAST to download data products.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: astropy, astroquery, numpy, (yaml)
\n", + "**Tools**: astropy, astroquery, numpy, os, glob, (yaml)
\n", "**Cross-instrument**: all
\n", "\n", "**Content**\n", @@ -45,7 +45,9 @@ "from astropy.coordinates import SkyCoord\n", "from astropy.io import fits\n", "from astroquery.mast import Observations\n", - "import numpy as np" + "import numpy as np\n", + "import os\n", + "import glob" ] }, { @@ -87,7 +89,7 @@ }, { "cell_type": "markdown", - "id": "ded88209", + "id": "d47efd71", "metadata": {}, "source": [ "#### Search with Proposal ID" @@ -117,7 +119,7 @@ }, { "cell_type": "markdown", - "id": "28b20df5", + "id": "4d358734", "metadata": {}, "source": [ "#### Search with Observation ID\n", @@ -206,7 +208,7 @@ }, { "cell_type": "markdown", - "id": "2109ecff", + "id": "179f01e4", "metadata": {}, "source": [ "#### Filtering" @@ -340,6 +342,60 @@ " break # comment this out if you want to download everything" ] }, + { + "cell_type": "markdown", + "id": "a1eb6450", + "metadata": {}, + "source": [ + "## Move files to preferred location\n", + "\n", + "At this point, I move the rate files and the association files to a preferred place on my machine:\n", + "- all rate files under NGDEEP/rate\n", + "- level 2 association files under NGDEEP/asn_level2\n", + "- level 3 association files under NGDEEP/asn_level3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5688a317", + "metadata": {}, + "outputs": [], + "source": [ + "# first, make all of the new directories\n", + "topdir = 'data/NGDEEP'\n", + "if not os.path.exists(topdir):\n", + " os.mkdir(topdir)\n", + " print('Created:', topdir)\n", + "\n", + "for subdir in ['rate', 'asn_level2', 'asn_level3']:\n", + " new_subdir = os.path.join(topdir, subdir)\n", + " if not os.path.exists(new_subdir):\n", + " os.mkdir(new_subdir)\n", + " print('Created:', new_subdir)\n", + "\n", + "file_dict = {'rate' : 'rate',\n", + " 'image3' : 'asn_level3',\n", + " 'image2' : 'asn_level2'}\n", + "\n", + "# now move all of the files to the appropriate locations\n", + "for filename in glob.glob('data/*.fits') + glob.glob('data/*.json'):\n", + " try:\n", + " index = filename.split('_')[2] # json files; looking for image2/image3\n", + " subdir = file_dict[index]\n", + " except KeyError:\n", + " try:\n", + " index2 = filename.split('_')[-1].split('.')[0] # rate files\n", + " subdir = file_dict[index2]\n", + " except KeyError:\n", + " print(f'Unrecognized index: {index} or {index2}')\n", + " continue\n", + " \n", + " new_file = os.path.join(topdir, subdir, os.path.basename(filename))\n", + " os.rename(filename, new_file)\n", + " print(f'Moved: {filename} to {new_file}')" + ] + }, { "cell_type": "markdown", "id": "8c0e1d71", From 4934c959100a31eea7098615fbec165332acfa10 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 16 Oct 2023 14:47:04 -0400 Subject: [PATCH 11/62] Merging part of notebook 01 into 03. Also adding comments and non manual way to match direct to dispersed images --- .../03_imviz_level2.ipynb | 160 +++++++++++++++--- 1 file changed, 140 insertions(+), 20 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb b/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb index fafa999b8..881aae2c8 100644 --- a/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb @@ -19,11 +19,14 @@ "source": [ "from jdaviz import Imviz\n", "import warnings\n", - "from astropy.io import ascii\n", + "from astropy.io import ascii, fits\n", "from astropy.table import Table\n", "from astropy.coordinates import SkyCoord, Angle\n", "import astropy.units as u\n", - "import numpy as np" + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import glob" ] }, { @@ -34,6 +37,49 @@ "## Read list of files" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "05afdbfc", + "metadata": {}, + "outputs": [], + "source": [ + "def write_keywords_csv(datadir):\n", + " # set up an empty dataframe initially to fill in\n", + " df = pd.DataFrame(columns=[\"FILENAME\",\n", + " \"TARG_RA\", \n", + " \"TARG_DEC\", \n", + " \"FILTER\", # Grism; GR150R/GR150C\n", + " \"PUPIL\", # Filter used; F090W, F115W, F140M, F150W F158M, F200W\n", + " \"PATT_NUM\", # Position number within dither pattern for WFSS\n", + " \"NUMDTHPT\", # Total number of points in entire dither pattern\n", + " \"XOFFSET\", # X offset from pattern starting position for NIRISS (arcsec)\n", + " \"YOFFSET\"]) # Y offset from pattern starting position for NIRISS (arcsec)\n", + "\n", + " for ratefile in glob.glob(os.path.join(datadir, \"*rate.fits\")):\n", + " image = fits.open(ratefile)\n", + "\n", + " # create a new dataframe for each file\n", + " df2 = pd.DataFrame({\"FILENAME\" : ratefile,\n", + " \"TARG_RA\" : [image[0].header[\"TARG_RA\"]],\n", + " \"TARG_DEC\" : [image[0].header[\"TARG_DEC\"]],\n", + " \"FILTER\" : [image[0].header[\"FILTER\"]],\n", + " \"PUPIL\" : [image[0].header[\"PUPIL\"]],\n", + " \"PATT_NUM\" : [image[0].header[\"PATT_NUM\"]],\n", + " \"NUMDTHPT\" : [image[0].header[\"NUMDTHPT\"]],\n", + " \"XOFFSET\" : [image[0].header[\"XOFFSET\"]],\n", + " \"YOFFSET\" : [image[0].header[\"YOFFSET\"]]\n", + " })\n", + "\n", + " # merge the two dataframes together to create a dataframe with all \n", + " df = pd.concat([df, df2], ignore_index=True, axis=0)\n", + "\n", + " dfsort = df.sort_values('FILENAME', ignore_index=False)\n", + "\n", + " dfsort.to_csv(\"./list_ngdeep_rate.csv\", sep=',', index=False)\n", + " return dfsort" + ] + }, { "cell_type": "code", "execution_count": null, @@ -42,8 +88,23 @@ "outputs": [], "source": [ "listrate_file = './list_ngdeep_rate.csv'\n", - "listrate = Table.read(listrate_file)\n", - "listrate[46::]" + "\n", + "if os.path.exists(listrate_file):\n", + " rate_df = Table.read(listrate_file)\n", + "else:\n", + " rate_df = write_keywords_csv('data/NGDEEP/rate')\n", + "\n", + "rate_df[-1::]" + ] + }, + { + "cell_type": "markdown", + "id": "4d7f78b7", + "metadata": {}, + "source": [ + "While you could look at the rate images, instead consider running the files through flatfield step of the pipeline to clean up detector artifacts.\n", + "\n", + "** Insert notebook 02 here as an optional function and call it. Fix the function to use call instead while doing this, too ** " ] }, { @@ -57,20 +118,15 @@ { "cell_type": "code", "execution_count": null, - "id": "352872bc", + "id": "7e26b8a3", "metadata": {}, "outputs": [], "source": [ - "filedir = '/Users/cpacifici/DATA/NGDEEP/wcs_flat/'\n", - "clearim = []\n", - "datalabel = []\n", - "for ii, file in enumerate(listrate['FILENAME']):\n", - " if listrate['FILTER'][ii]=='CLEAR':\n", - " name = file.split('/')[-1]\n", - " newname = name.replace('rate.fits', 'flatfieldstep.fits')\n", - " clearim.append(filedir+newname)\n", - " datalabel.append(listrate['PUPIL'][ii])\n", - "clearim" + "direct_image_df = rate_df[rate_df['FILTER'] == 'CLEAR'] # selecting out only the direct images\n", + "clearim = direct_image_df['FILENAME']\n", + "# optional flatfield step would be:\n", + "flatfield_im = [f.replace('rate.fits', 'flatfieldstep.fits') for f in clearim]\n", + "datalabel = direct_image_df['PUPIL']" ] }, { @@ -79,7 +135,9 @@ "metadata": {}, "source": [ "### Use Imviz batch load since it is many images\n", - "It takes about 1 minute to load all the images." + "It takes about 1 minute to load all the images.\n", + "\n", + "** I need some basic instructions here to know how to navigate **" ] }, { @@ -92,10 +150,10 @@ "outputs": [], "source": [ "imviz = Imviz()\n", - "for ii in range(len(clearim)):\n", + "for img, filt_label in zip(clearim, datalabel):\n", " with warnings.catch_warnings():\n", " warnings.simplefilter('ignore')\n", - " imviz.load_data(clearim[ii], data_label=datalabel[ii])\n", + " imviz.load_data(img, data_label=filt_label)\n", "imviz.show()" ] }, @@ -161,7 +219,66 @@ "metadata": {}, "source": [ "## Check direct images and their corresponding WFSS images\n", - "Selecting by hand from the table to use direct and dispersed images on the same dither position." + "\n", + "Look at direct and dispersed images on the same dither position." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa07c90d", + "metadata": {}, + "outputs": [], + "source": [ + "# Using the XOFFSET and YOFFSET keywords, match the direct image and dispersed (grism) images to each other\n", + "\n", + "file_dict = {}\n", + "for xoff, yoff, imgname in zip(direct_image_df['XOFFSET'], direct_image_df['YOFFSET'], \n", + " direct_image_df['FILENAME']):\n", + " match_pointing_df = rate_df[(rate_df['XOFFSET'] == xoff) & \n", + " (rate_df['YOFFSET'] == yoff) &\n", + " (rate_df['FILENAME'] != imgname)]\n", + " \n", + " # If there aren't any matches, keep moving\n", + " if not len(match_pointing_df):\n", + " continue\n", + " \n", + " # fill in the direct / dispersed dataframe for each of the filters\n", + " temp_filter = match_pointing_df.iloc[0]['PUPIL']\n", + " if temp_filter in list(file_dict.keys()):\n", + " # For now, only grab the first instance of the image/grism pair\n", + " continue\n", + " else:\n", + " print('**', temp_filter, imgname)\n", + " print(match_pointing_df.iloc[0]['FILENAME'])\n", + " print(match_pointing_df.iloc[0]['FILTER'])\n", + " file_dict[temp_filter] = {'direct_img' : imgname,\n", + " 'dispersed_img' : match_pointing_df.iloc[0]['FILENAME']}\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "963c1f6a", + "metadata": {}, + "outputs": [], + "source": [ + "imviz2 = Imviz()\n", + "with warnings.catch_warnings():\n", + " warnings.simplefilter('ignore')\n", + " for filt, img_dict in file_dict.items(): # loop over all filters\n", + " for imgtype, filename in img_dict.items(): # loop over all files\n", + " imviz2.load_data(filename, data_label=f\"{filt} {imgtype}\")\n", + "imviz2.show()" + ] + }, + { + "cell_type": "markdown", + "id": "d4273b91", + "metadata": {}, + "source": [ + "#### Manual Check" ] }, { @@ -171,10 +288,13 @@ "metadata": {}, "outputs": [], "source": [ + "filedir = 'data/NGDEEP/rate/'\n", "dir_f115w = filedir + 'jw02079004001_04101_00004_nis_flatfieldstep.fits'\n", "disp_f115w = filedir + 'jw02079004001_05101_00001_nis_flatfieldstep.fits'\n", + "\n", "dir_f150w = filedir + 'jw02079004002_10101_00004_nis_flatfieldstep.fits'\n", "disp_f150w = filedir + 'jw02079004002_11101_00001_nis_flatfieldstep.fits'\n", + "\n", "dir_f200w = filedir + 'jw02079004003_04101_00004_nis_flatfieldstep.fits'\n", "disp_f200w = filedir + 'jw02079004003_05101_00001_nis_flatfieldstep.fits'" ] @@ -245,7 +365,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.9.12" } }, "nbformat": 4, From b03af501f13284b39ccd76e6fe7083f342ff0a09 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 16 Oct 2023 14:47:50 -0400 Subject: [PATCH 12/62] Changing env call to be more generic --- .../02_run_pipeline_wcs_flatfield.ipynb | 43 +++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb b/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb index 9cbee229c..0251f1de1 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb @@ -8,6 +8,31 @@ "# Run assign_wcs and flatfielding" ] }, + { + "cell_type": "markdown", + "id": "5e0e0498", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "id": "da56a9fb", + "metadata": {}, + "source": [ + "\n", + "## Imports" + ] + }, + { + "cell_type": "raw", + "id": "58db78a6", + "metadata": {}, + "source": [ + "# Update the CRDS path to your local directory\n", + "%env CRDS_PATH= (/grp/crds/jwst/)\n", + "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" + ] + }, { "cell_type": "code", "execution_count": null, @@ -15,10 +40,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Update the CRDS path to your local directory\n", - "%env CRDS_PATH=/Users/cpacifici/crds_cache_july2023\n", - "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu\n", - " \n", "import os\n", "import glob\n", "#jwst pipeline related modules\n", @@ -42,6 +63,12 @@ "print('jwst:', jwst.__version__)" ] }, + { + "cell_type": "markdown", + "id": "aa57e27d", + "metadata": {}, + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -49,9 +76,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Update appropriately (got the rate files from MAST)\n", - "data_dir_in = \"/Users/cpacifici/DATA/NGDEEP/rate/*\"\n", - "data_dir_out = \"/Users/cpacifici/DATA/NGDEEP/wcs_flat/\"" + "# Directory definitions. The rate files should be located here based on notebooks 00 & 01\n", + "data_dir_in = \"data/NGDEEP/rate/*\"\n", + "data_dir_out = \"data/NGDEEP/wcs_flat/\"" ] }, { @@ -156,7 +183,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.9.12" } }, "nbformat": 4, From e14be9c2d5f116209f9fb25214c2a9c26a14594b Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Tue, 23 Jan 2024 18:04:26 -0500 Subject: [PATCH 13/62] minor documentation updates and print statements --- .../00_mast_query_progID.ipynb | 77 ++++++++++++------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb index c6793817a..533b20e9b 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "source": [ "# Get observations from program ID\n", - "This notebook uses the python [astroquery.mast Observations](https://astroquery.readthedocs.io/en/latest/mast/mast_obsquery.html) class of the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three NIRISS filters: F115W, F150W, and F200W using both GR150R and GR150C grisms.\n", + "This notebook uses the python [astroquery.mast Observations](https://astroquery.readthedocs.io/en/latest/mast/mast_obsquery.html) class of the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three [NIRISS filters](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrumentation/niriss-pupil-and-filter-wheels): F115W, F150W, and F200W using both GR150R and GR150C grisms.\n", "\n", "**Use case**: use MAST to download data products.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", @@ -15,12 +15,17 @@ "\n", "**Content**\n", "- [Imports](#imports)\n", - "- [Set up the query and the batches](#setup)\n", - "- [Filter and download products](#filter)\n", + "- [Querying for Observations](#setup)\n", + " - [Search with Proposal ID](#propid)\n", + " - [Search with Observation ID](#obsid)\n", + "- [Filter and Download Products](#filter)\n", + " - [Filtering Data Before Downloading](#filter_data)\n", + " - [Downloading Data](#downloading)\n", + "- [Reorganize Directory Structure](#reorg)\n", "\n", "\n", - "**Author**: Camilla Pacifici (cpacifici@stsci.edu) & Rachel Plesha (rplesha@stsci.edu)\n", - "**Last modified**: October 2023\n", + "**Author**: Camilla Pacifici (cpacifici@stsci.edu) & Rachel Plesha (rplesha@stsci.edu)
\n", + "**Last modified**: January 2024\n", "\n", "This notebook was inspired by the [JWebbinar session about MAST](https://github.com/spacetelescope/jwebbinar_prep/blob/main/mast_session/Crowded_Field/Crowded_Field.ipynb)." ] @@ -36,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "7e92dee7", "metadata": {}, "outputs": [], @@ -56,7 +61,7 @@ "metadata": {}, "source": [ "\n", - "## Set up the query and the batches" + "## Querying for Observations" ] }, { @@ -92,6 +97,7 @@ "id": "d47efd71", "metadata": {}, "source": [ + "\n", "#### Search with Proposal ID" ] }, @@ -113,8 +119,8 @@ " )\n", "\n", "print(len(obs_table), 'files found')\n", - "# look at what was obtained in this query\n", - "obs_table" + "# look at what was obtained in this query for a select number of column names of interest\n", + "obs_table[['obs_collection', 'instrument_name', 'filters', 'target_name', 'obs_id', 's_ra', 's_dec', 't_exptime', 'proposal_id']]" ] }, { @@ -122,6 +128,7 @@ "id": "4d358734", "metadata": {}, "source": [ + "\n", "#### Search with Observation ID\n", "The observation ID (obs_id) allows for flexibility of searching by the proposal ID and the observation ID because of how the JWST filenames are structured. More information about the JWST file naming conventions can be found at: https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/file_naming.html.\n", "\n", @@ -150,7 +157,7 @@ "metadata": {}, "source": [ "\n", - "## Filter and download products" + "## Filter and Download Products" ] }, { @@ -165,9 +172,7 @@ "cell_type": "code", "execution_count": null, "id": "b8a653a9", - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "batch_size = 5 # 5 files at a time maximizes the download speed.\n", @@ -177,8 +182,8 @@ "print(\"How many batches?\", len(obs_batches))\n", "\n", "single_group = obs_batches[0] # Useful to inspect the files obtained from one group\n", - "print(\"Inspect the first batch:\")\n", - "single_group" + "print(\"Inspect the first batch to ensure that it matches expectations of what you want downloaded:\")\n", + "single_group['obs_collection', 'instrument_name', 'filters', 'target_name', 'obs_id', 's_ra', 's_dec', 't_exptime', 'proposal_id']" ] }, { @@ -211,7 +216,8 @@ "id": "179f01e4", "metadata": {}, "source": [ - "#### Filtering" + "\n", + "#### Filtering Data Before Downloading" ] }, { @@ -273,10 +279,10 @@ "id": "e98933e6", "metadata": {}, "source": [ - "We can see that for each exposure in the observation list, there are many files associated that need to be downloaded. This is why we need to work in batches to download.\n", - "\n", + "From above, we can see that for each exposure name in the observation list (`obs_table`), there are many associated files in the background that need to be downloaded as well. This is why we need to work in batches to download.\n", "\n", - "#### Downloading\n", + "\n", + "#### Downloading Data\n", "To actually download the products, provide ``Observations.download_products()`` with a list of the filtered products. \n", "\n", "If the data are proprietary, you may also need to set up your API token. **NEVER** commit your token to a public repository. An alternative is to create a separate configuration file (config_file.yaml) that is readable only to you and has a key: 'mast_token' : _API token_\n", @@ -290,7 +296,7 @@ "id": "fc1897a3", "metadata": {}, "source": [ - "# if needed, replace Observations.download_products() with the following:\n", + "# if needed, create a separate configuration file and replace Observations.download_products() with the following:\n", "\n", "import yaml\n", "\n", @@ -310,11 +316,13 @@ }, "outputs": [], "source": [ + "download_dir = 'data'\n", + "\n", "# Now let's get the products for each batch of observations, and filter down to only the products of interest.\n", "for index, batch in enumerate(obs_batches):\n", " \n", " # Progress indicator...\n", - " print('\\n'+f'Batch #{index+1}')\n", + " print('\\n'+f'Batch #{index+1} / {len(obs_batches)}')\n", " \n", " # Make a list of the `obsid` identifiers from our Astropy table of observations to get products for.\n", " obsids = batch['obsid']\n", @@ -333,23 +341,35 @@ " )\n", " # Download products for these records.\n", " manifest = Observations.download_products(filtered_products,\n", - " download_dir='data',\n", + " download_dir=download_dir,\n", " flat=True, # astroquery v0.4.7 or later only\n", " ) \n", - " #print('Products downloaded:\\n', filtered_products['productFilename'])\n", + " print('Products downloaded:\\n', filtered_products['productFilename'])\n", " \n", " # only downloading the first batch of 5 observations\n", " break # comment this out if you want to download everything" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "39310ec6-f58c-4dd6-96ba-b5cda55ee7d5", + "metadata": {}, + "outputs": [], + "source": [ + "downloaded_files = glob.glob(os.path.join(download_dir, '*.fits')) + glob.glob(os.path.join(download_dir, '*.json'))\n", + "print(len(downloaded_files), 'files downloaded to:', download_dir)" + ] + }, { "cell_type": "markdown", "id": "a1eb6450", "metadata": {}, "source": [ - "## Move files to preferred location\n", + "\n", + "## Reorganize Directory Structure\n", "\n", - "At this point, I move the rate files and the association files to a preferred place on my machine:\n", + "This section takes the downloaded data and sorts it into a slightly different file structure to work with later in the notebooks. The expected format for this section is that all downloaded data are in a single directory (`flat=True`). The new file stucture is as follows:\n", "- all rate files under NGDEEP/rate\n", "- level 2 association files under NGDEEP/asn_level2\n", "- level 3 association files under NGDEEP/asn_level3" @@ -375,13 +395,16 @@ " print('Created:', new_subdir)\n", "\n", "file_dict = {'rate' : 'rate',\n", + " 'image2' : 'asn_level2',\n", + " 'spec2' : 'asn_level2',\n", " 'image3' : 'asn_level3',\n", - " 'image2' : 'asn_level2'}\n", + " 'spec3' : 'asn_level3',\n", + " }\n", "\n", "# now move all of the files to the appropriate locations\n", "for filename in glob.glob('data/*.fits') + glob.glob('data/*.json'):\n", " try:\n", - " index = filename.split('_')[2] # json files; looking for image2/image3\n", + " index = filename.split('_')[2] # json files; looking for image2/image3 or spec2/spec3\n", " subdir = file_dict[index]\n", " except KeyError:\n", " try:\n", From 16328ea950e827e2bbecdd93bd905b7c7a3eb7de Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 29 Jan 2024 15:17:17 -0500 Subject: [PATCH 14/62] Reworking the notebook with more documentation and examples --- .../04_run_pipeline_imaging_level2and3.ipynb | 485 ++++++++++-------- 1 file changed, 260 insertions(+), 225 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb index 0ffb50fb6..effa06c91 100644 --- a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb @@ -5,9 +5,36 @@ "id": "7a935d7c", "metadata": {}, "source": [ - "# Run pipeline and create catalog of sources\n", + "# Run Image pipeline and create catalog of sources\n", "\n", - "This runs the imaging pipeline. Credit JWebbinar notebooks." + "The first calibration that should be done as part of a WFSS run is to run the direct images through the Image2 and Image3 steps of the JWST pipeline. This includes creating a source catalog, which most likely will need to be adjusted from the pipeline default values. **Not having a good source catalog will result in non optimal extraction of sources in the dispersed, WFSS, images.**\n", + "\n", + "**Use case**: The default parameters for the pipeline do not extract the expected sources, so a custom parameters need to be set to obtain new combined image and source catalog.
\n", + "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", + "**Tools**: glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, shutil, warnings
\n", + "**Cross-instrument**: NIRISS
\n", + "\n", + "**Content**\n", + "- [Imports & Data Setup](#imports)\n", + "- [Default Imaging Pipeline Run](#default)\n", + " - [Image2](#default_image2)\n", + " - [Image3](#default_image3)\n", + " - [Inspecting Default Results](#view_default)\n", + "- [Custom Imaging Pipeline Run]\n", + " - [Image3]\n", + " - [Inspecting Custom Results]\n", + "\n", + "**Author**: Camilla Pacifici (cpacifici@stsci.edu), Rachel Plesha (rplesha@stsci.edu), JWebbinar notebooks.
\n", + "**Last modified**: January 2024" + ] + }, + { + "cell_type": "markdown", + "id": "7fdaf1a1-57e5-4239-b82e-6552261d8d82", + "metadata": {}, + "source": [ + "\n", + "## Imports & Data Setup" ] }, { @@ -18,57 +45,61 @@ "outputs": [], "source": [ "import os\n", - "os.environ[\"CRDS_PATH\"] = \"/Users/cpacifici/crds_cache_july2023/\" # set appropriate path\n", - "os.environ[\"CRDS_SERVER_URL\"] = \"https://jwst-crds.stsci.edu\"\n", - "\n", "import glob\n", + "import json\n", "import shutil\n", - "\n", - "import jwst\n", - "from jwst.pipeline import Image2Pipeline\n", - "from jwst.pipeline import Image3Pipeline\n", - "\n", - "from astropy.table import Table\n", - "\n", + "import warnings\n", "import numpy as np\n", - "\n", - "import json\n", - "\n", - "from jdaviz import Imviz\n", - "import warnings" + "import pandas as pd\n", + "from astropy.io import fits\n", + "from astropy.table import Table\n", + "from matplotlib import pyplot as plt\n", + "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, - "id": "ae10a560", + "id": "bd8dc296-44cc-4118-a036-5a9ac68c456f", "metadata": {}, "outputs": [], "source": [ - "print('jwst:', jwst.__version__)" + "import jwst\n", + "from jwst.pipeline import Image2Pipeline\n", + "from jwst.pipeline import Image3Pipeline\n", + "from jdaviz import Imviz" + ] + }, + { + "cell_type": "raw", + "id": "42888b91-e010-4ed2-b101-f586668a2235", + "metadata": {}, + "source": [ + "# Update the CRDS path to your local directory\n", + "%env CRDS_PATH= (/grp/crds/jwst/)\n", + "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" ] }, { "cell_type": "code", "execution_count": null, - "id": "60437b9c", + "id": "ae10a560", "metadata": {}, "outputs": [], "source": [ - "# Update appropriately\n", - "data_dir_in = \"/Users/cpacifici/DATA/NGDEEP/rate/\"\n", - "data_dir_asn_level2 = \"/Users/cpacifici/DATA/NGDEEP/asn_level2/\"\n", - "data_dir_asn_level3 = \"/Users/cpacifici/DATA/NGDEEP/asn_level3/\"\n", - "data_dir_out_image2 = \"/Users/cpacifici/DATA/NGDEEP/image2/\"\n", - "data_dir_out_image3 = \"/Users/cpacifici/DATA/NGDEEP/image3/\"" + "print('jwst:', jwst.__version__)" ] }, { - "cell_type": "markdown", - "id": "77a8ea4e", + "cell_type": "code", + "execution_count": null, + "id": "e2ecc213-479a-422c-a0f8-8608309b845c", "metadata": {}, + "outputs": [], "source": [ - "## Read list of rate files for imaging" + "data_dir = 'data'\n", + "default_run_image3 = 'default_image3_calibrated' # where the results of the default image3 run will be saved (inside of data_dir)\n", + "custom_run_image3 = 'custom_image3_calibrated'# where the results of the custom image3 run will be saved (inside of data_dir)" ] }, { @@ -79,139 +110,138 @@ "outputs": [], "source": [ "listrate_file = './list_ngdeep_rate.csv'\n", - "listrate = Table.read(listrate_file)\n", - "listrate[np.where(listrate['FILTER']=='CLEAR')]" + "rate_df = pd.read_csv(listrate_file)" ] }, { "cell_type": "markdown", - "id": "0ac742e8", + "id": "d991e323-8e98-4487-a8ab-1b696f91899b", "metadata": {}, "source": [ - "## Run image2 on all CLEAR images using the provided association files" + "The association files expect that 1) all of the data are in the same directory and 2) that you are performing the pipeline call also in that directory. Because of that, we need to change into the data directory to run the imaging pipelines." ] }, { - "cell_type": "markdown", - "id": "82eb2c93", + "cell_type": "code", + "execution_count": null, + "id": "a4ea2cb0-e5d5-4347-8c4c-9126cda54561", "metadata": {}, + "outputs": [], "source": [ - "### Check an association file for level 2" + "cwd = os.getcwd() # get the current working directory \n", + "if cwd != data_dir: # if you are not already in the location of the data, change into it\n", + " try:\n", + " os.chdir(data_dir)\n", + " except FileNotFoundError:\n", + " print(f\"Not able to change into: {data_dir}.\\nRemaining in: {cwd}\")\n", + " pass" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "74bc920c", + "cell_type": "markdown", + "id": "10824c6a-4843-4f31-beb7-05ecc58e8b84", "metadata": {}, - "outputs": [], "source": [ - "asnfile = data_dir_asn_level2 + 'jw02079-o004_20230622t175524_image2_00001_asn.json'\n", - "asn_data = json.load(open(asnfile))\n", - "print(asn_data['products'][0]['members'][0]['expname'])\n", - "asn_data" + "\n", + "## Default Imaging Pipeline Run\n", + "To start, run the default image2 and image3 steps of the pipeline on all direct images observed with the WFSS data." ] }, { "cell_type": "markdown", - "id": "3cf2e472", + "id": "0ac742e8", "metadata": {}, "source": [ - "**Developer note:** I do not see a way to tell which association file looks at which rate file without opening it and I believe association files and rate files need to be in the same directory where the pipeline is running, so I am going to temporarily copy all association file and all rate files in here and delete them when image2 is done." + "\n", + "### Run Default Image2\n", + "\n", + "Image2 is run on the direct image rate files. While your program should have valid association files to download from MAST, if for any reason you need to make your own association file, see [Creating Custom ASN Files](#customasn)." ] }, { "cell_type": "markdown", - "id": "0421c67b", + "id": "82eb2c93", "metadata": {}, "source": [ - "### Copy all needed files to the current directory" + "#### Looking in a Level 2 Imaging Association File\n", + "First, take a look inside the association (ASN) files to better understand everything that is contained in them." ] }, { "cell_type": "code", "execution_count": null, - "id": "eb0a1344", + "id": "f6429ae4-cc03-4718-b2c2-12924170fe89", "metadata": {}, "outputs": [], "source": [ - "asn_image2 = glob.glob(data_dir_asn_level2+'*image2*')\n", - "for file in asn_image2:\n", - " shutil.copy(file, './')" + "image2_asns = glob.glob('*image2*asn*.json')\n", + "print(len(image2_asns), 'Image2 ASN files')" ] }, { "cell_type": "code", "execution_count": null, - "id": "c62185ff", + "id": "9226e1b9-9553-402b-910d-73f7cffcb784", "metadata": {}, "outputs": [], "source": [ - "rate_clear = listrate[np.where(listrate['FILTER']=='CLEAR')]['FILENAME']\n", - "for file in rate_clear:\n", - " shutil.copy(data_dir_in+file, './')" + "## look at one of the association files\n", + "asn_data = json.load(open(image2_asns[0]))\n", + "for key, data in asn_data.items():\n", + " print(f\"{key} : {data}\")" ] }, { "cell_type": "markdown", - "id": "eaa48cf9", + "id": "99c3a7f0-ec0a-403a-9c55-58b2ac6559a7", "metadata": {}, "source": [ - "### Run image2" + "From this association, we can tell many things about the observation:\n", + "1. From `asn_type` and `asn_rule`, we can see that this is an image2 association\n", + "2. From `degraded_status` we can see that there are no exposures to not be included in the calibration.\n", + "3. From `constraints`, we can see this is not a time series observation (TSO), the observation is part of program 2079, observed with NIRISS with the CLEAR (i.e. imaging for WFSS) and F150W filters.\n", + "4. From `products` we can see there is only one exposure associated" ] }, { "cell_type": "code", "execution_count": null, - "id": "373ccb5a", + "id": "e5641931-1042-4494-a527-43e521803f3c", "metadata": {}, "outputs": [], "source": [ - "asn_image2_local = glob.glob('./jw02079-o004_*_image2_*_asn.json')\n", - "asn_image2_local" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dff099be", - "metadata": {}, - "outputs": [], - "source": [ - "for file in asn_image2_local:\n", - " print(file)\n", - " image2 = Image2Pipeline()\n", - " result = image2.call(file,\n", - " save_results=True,\n", - " output_dir=data_dir_out_image2) # Use the defaults for now" + "## in particular, take a closer look at the product filenames with the association file:\n", + "for product in asn_data['products']:\n", + " for key, value in product.items():\n", + " if key == 'members':\n", + " print(f\"{key}:\")\n", + " for member in value:\n", + " print(f\" {member['expname']} {member['exptype']}\")\n", + " else:\n", + " print(f\"{key}: {value}\")" ] }, { "cell_type": "markdown", - "id": "1bddeaa8", + "id": "db2dbaea-228e-4565-928b-2984f4d7fda7", "metadata": {}, "source": [ - "### Delete association and rate files from local directory" + "#### Run image2\n", + "\n", + "The `rate.fits` products will be calibrated into `cal.fits` files. More information about the steps performed in the Image2 part of the pipeline can be found in the [Image2 pipeline documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_image2.html).\n", + "\n", + "In this case, we're saving the outputs to the same directory we are running the pipeline in so that we can then use the output `cal` files to run the Image3 pipeline" ] }, { "cell_type": "code", "execution_count": null, - "id": "7caa5b04", + "id": "dff099be", "metadata": {}, "outputs": [], "source": [ - "dir_to_be_del = './files_to_be_deleted'\n", - "if not os.path.exists(dir_to_be_del):\n", - " os.mkdir(dir_to_be_del)\n", - " \n", - "for file in asn_image2_local:\n", - " shutil.move(file, dir_to_be_del)\n", - " \n", - "for file in glob.glob('./*rate.fits'):\n", - " shutil.move(file, dir_to_be_del)\n", - " \n", - "shutil.rmtree(dir_to_be_del)" + "for img2_asn in image2_asns:\n", + " img2 = Image2Pipeline.call(img2_asn, save_results=True)" ] }, { @@ -219,7 +249,8 @@ "id": "fe1ae678", "metadata": {}, "source": [ - "## Run image3 for the three filters and create catalogs" + "\n", + "### Run Default Image3 " ] }, { @@ -227,59 +258,66 @@ "id": "e1fd5c10", "metadata": {}, "source": [ - "### Check an association file for level 3" + "#### Looking in a Level 3 Association File\n", + "The contents are quite similar to image2, but notice now that there are many more members that are associated together, and they use the `cal.fits` files from image2." ] }, { "cell_type": "code", "execution_count": null, - "id": "add4839e", + "id": "ccf6ac3b-333f-451d-8157-f83ccba9352c", "metadata": {}, "outputs": [], "source": [ - "asnfile = data_dir_asn_level3 + 'jw02079-o004_20230622t175524_image3_00002_asn.json'\n", - "asn_data = json.load(open(asnfile))\n", - "asn_data" + "image3_asns = glob.glob('*image3*asn*.json')\n", + "print(len(image3_asns), 'Image3 ASN files')" ] }, { - "cell_type": "markdown", - "id": "2f4dd3b2", + "cell_type": "code", + "execution_count": null, + "id": "add4839e", "metadata": {}, + "outputs": [], "source": [ - "### Move all needed files to the current directory" + "## look at one of the association files\n", + "image3_asn_data = json.load(open(image3_asns[0]))\n", + "for key, data in image3_asn_data.items():\n", + " print(f\"{key} : {data}\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "97e5a055", + "id": "2831ae47-5b1b-4d92-b02a-8ce1ee359710", "metadata": {}, "outputs": [], "source": [ - "asn_image3 = glob.glob(data_dir_asn_level3+'*image3*')\n", - "for file in asn_image3:\n", - " shutil.copy(file, './')" + "## in particular, take a closer look at the product filenames with the association file:\n", + "for product in image3_asn_data['products']:\n", + " for key, value in product.items():\n", + " if key == 'members':\n", + " print(f\"{key}:\")\n", + " for member in value:\n", + " print(f\" {member['expname']} {member['exptype']}\")\n", + " else:\n", + " print(f\"{key}: {value}\")" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "faddcfe7", + "cell_type": "markdown", + "id": "0cab171d", "metadata": {}, - "outputs": [], "source": [ - "cal_clear = glob.glob(data_dir_out_image2+'*')\n", - "for file in cal_clear:\n", - " shutil.copy(file, './')" + "#### Run image3" ] }, { "cell_type": "markdown", - "id": "0cab171d", + "id": "ca3bd322-7f67-43c4-8dae-dc47779462ee", "metadata": {}, "source": [ - "### Run image3" + "Image3 is where we can make some adjustments to obtain a better output source catalog. The `cal.fits` files will be calibrated into a single combined `i2d.fits` image. More information about the steps performed in the Image3 part of the pipeline can be found in the [Image3 pipeline documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_image3.html)." ] }, { @@ -289,8 +327,8 @@ "metadata": {}, "outputs": [], "source": [ - "asn_image3_local = glob.glob('./*image3*.json')\n", - "asn_image3_local" + "for img3_asn in image3_asns:\n", + " img3 = Image3Pipeline.call(img3_asn, save_results=True, output_dir=default_run_image3)" ] }, { @@ -332,128 +370,138 @@ }, { "cell_type": "markdown", - "id": "e7618691", + "id": "33f0dbeb-4caf-4d97-8b2c-3255f906e76b", "metadata": {}, "source": [ - "### Inspect the catalogs used for tweakreg" + "\n", + "### Inspecting Default Results" ] }, { "cell_type": "code", "execution_count": null, - "id": "64bf95eb", + "id": "47da18f6-92e3-4fff-8153-222cfc55d353", "metadata": {}, "outputs": [], "source": [ - "cal_files = glob.glob(data_dir_out_image2+'*')\n", - "cat_files = []\n", - "for file in cal_files:\n", - " filename = file.split('/')[-1]\n", - " catname = filename.replace('cal.fits', 'cal_cat.ecsv')\n", - " cat_files.append(catname)" + "# These are all resuts from the Image3 pipeline\n", + "image3_i2d = np.sort(glob.glob(os.path.join(default_run_image3, '*i2d.fits'))) # combined image over multiple dithers/mosaic\n", + "image3_segm = np.sort(glob.glob(os.path.join(default_run_image3, '*segm.fits'))) # segmentation map that defines the extent of a source\n", + "image3_cat = np.sort(glob.glob(os.path.join(default_run_image3, '*cat.ecsv'))) # Source catalog that defines the RA/Dec of a source at a particular pixel" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "8830aadb", + "cell_type": "markdown", + "id": "7442c52c-d399-4593-a488-e30e02261383", "metadata": {}, - "outputs": [], "source": [ - "calexposure = 2\n", - "imviz_cal = Imviz()\n", - "viewer = imviz_cal.default_viewer\n", - "imviz_cal.load_data(cal_files[calexposure])\n", - "plotopt = imviz_cal.plugins['Plot Options']\n", - "plotopt.stretch_preset = '99%'\n", - "imviz_cal.show()" + "#### Matplotlib\n", + "Matplotlib has limitations where ImViz might better suite your needs -- especially if you like to look at things in WCS coordinates. For the notebook purposes, we are highlighting a few key areas using the matplotlib package instead.\n", + "\n", + "Using the `i2d` combined image and the source catalog produced by Image3, we can visually inspect if we're happy with where the pipeline found the sources. In the following figures, what has been defined as an extended source by the pipeline is shown in orange-red, and what has been defined as a point source by the pipeline is shown in grey. This definition affects the extraction box in the WFSS images." ] }, { "cell_type": "code", "execution_count": null, - "id": "d2ce2462", + "id": "0945914a-bebc-4a8e-b619-fded2bd42485", "metadata": {}, "outputs": [], "source": [ - "catalog = Table.read(cat_files[calexposure])\n", - "t_xy = Table({'x': catalog['x'],\n", - " 'y': catalog['y']})\n", + "fig = plt.figure(figsize=(10, 10))\n", + "\n", + "cols = 2\n", + "rows = int(np.ceil(len(image3_i2d) / cols))\n", + "\n", + "for plt_num, img in enumerate(image3_i2d):\n", + "\n", + " # determine where the subplot should be\n", + " xpos = (plt_num%40) % cols\n", + " ypos = ((plt_num%40) // cols) # // to make it an int.\n", + "\n", + " # make the subplot\n", + " ax = plt.subplot2grid((rows, cols), (ypos, xpos))\n", "\n", - "viewer.marker = {'color': 'orange', 'alpha': 1, 'markersize': 20, 'fill': False}\n", - "viewer.add_markers(t_xy)" + " # plot the image\n", + " with fits.open(img) as hdu:\n", + " ax.imshow(hdu[1].data, vmin=0, vmax=0.3, origin='lower')\n", + " ax.set_title(f\"obs{hdu[0].header['OBSERVTN']} {hdu[0].header['PUPIL']}\")\n", + "\n", + " # also plot the associated catalog\n", + " cat = Table.read(img.replace('i2d.fits', 'cat.ecsv'))\n", + " extended_sources = cat[cat['is_extended'] == True]\n", + " point_sources = cat[cat['is_extended'] == False]\n", + " ax.scatter(extended_sources['xcentroid'], extended_sources['ycentroid'], s=20, facecolors='None', edgecolors='orangered', alpha=0.9)\n", + " ax.scatter(point_sources['xcentroid'], point_sources['ycentroid'], s=20, facecolors='None', edgecolors='dimgrey', alpha=0.9)\n", + "\n", + "# Helps to make the axes not overlap ; you can also set this manually if this doesn't work\n", + "plt.tight_layout()" ] }, { "cell_type": "markdown", - "id": "70e6116a", + "id": "d157e3a8-2f2a-4487-8329-f6fa56713212", "metadata": {}, "source": [ - "### Delete not-needed files from local directory\n", + "The segmentation maps are also a product of the Image3 pipeline, and they are used the help determine the source catalog. Let's take a look at those to ensure we are happy with what it is defining as a source.\n", "\n", - "List includes: cal, outlier_i2d, cal_cat, and asn." + "In the segmentation map, each yellow blob should correspond to a physical target. There are cases where sources can be blended, in which case the parameters for making the semgentation map and source catalog should be changed. An example of this can be seen below in the observation 004 F200W filter image where two galaxies have been blended into one source. This is discussed in more detail in [Custom Imaging Pipeline Run](#custom)." ] }, { "cell_type": "code", "execution_count": null, - "id": "5df25051", + "id": "b27845f3-65f0-46ce-9c0e-70e64ce65bcd", "metadata": {}, "outputs": [], "source": [ - "dir_to_be_del = './files_to_be_deleted'\n", - "if not os.path.exists(dir_to_be_del):\n", - " os.mkdir(dir_to_be_del)\n", - " \n", - "list_files = glob.glob('./jw02079*')\n", + "fig2 = plt.figure(figsize=(10, 20))\n", "\n", - "print(\"CHECK that list is ok before continuing\")\n", - "list_files" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c70a03f6", - "metadata": {}, - "outputs": [], - "source": [ - "for file in list_files:\n", - " shutil.move(file, dir_to_be_del)\n", + "cols = 2\n", + "rows = len(image3_i2d)\n", + "\n", + "for plt_num, img in enumerate(np.sort(np.concatenate([image3_segm, image3_i2d]))):\n", + "\n", + " # determine where the subplot should be\n", + " xpos = (plt_num%40) % cols\n", + " ypos = ((plt_num%40) // cols) # // to make it an int.\n", "\n", - "shutil.rmtree(dir_to_be_del)" + " # make the subplot\n", + " ax = plt.subplot2grid((rows, cols), (ypos, xpos))\n", + "\n", + " # plot the image\n", + " with fits.open(img) as hdu:\n", + " ax.imshow(hdu[1].data, vmin=0, vmax=0.3, origin='lower')\n", + " title = f\"obs{hdu[0].header['OBSERVTN']} {hdu[0].header['PUPIL']}\"\n", + "\n", + " # also plot the associated catalog\n", + " if 'i2d' in img:\n", + " cat = Table.read(img.replace('i2d.fits', 'cat.ecsv'))\n", + " ax.set_title(f\"{title} combined image\")\n", + " else:\n", + " ax.set_title(f\"{title} segmentation map\")\n", + "\n", + " extended_sources = cat[cat['is_extended'] == True]\n", + " point_sources = cat[cat['is_extended'] == False]\n", + " ax.scatter(extended_sources['xcentroid'], extended_sources['ycentroid'], s=20, facecolors='None', edgecolors='orangered', alpha=0.9)\n", + " ax.scatter(point_sources['xcentroid'], point_sources['ycentroid'], s=20, facecolors='None', edgecolors='dimgrey', alpha=0.9)\n", + " \n", + " # zooming in on a smaller region\n", + " ax.set_xlim(1250, 1750)\n", + " ax.set_ylim(1250, 1750)\n", + "\n", + "# Helps to make the axes not overlap ; you can also set this manually if this doesn't work\n", + "plt.tight_layout()" ] }, { "cell_type": "markdown", - "id": "bc3459a5", - "metadata": {}, - "source": [ - "## Visualize catalog and i2d images in Imviz" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c688aa3b", + "id": "d1addba0-ae56-4565-b12a-3ded03d9b0bf", "metadata": {}, - "outputs": [], "source": [ - "image3_i2d = [data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f115w_i2d.fits',\n", - " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f150w_i2d.fits',\n", - " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f200w_i2d.fits']\n", - "\n", - "image3_segm = [data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f115w_segm.fits',\n", - " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f150w_segm.fits',\n", - " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f200w_segm.fits']\n", + "#### ImViz\n", "\n", - "image3_cat = [data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv',\n", - " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f150w_cat.ecsv',\n", - " data_dir_out_image3 + 'jw02079-o004_t001_niriss_clear-f200w_cat.ecsv']\n", - "\n", - "image3_label = ['F115W',\n", - " 'F150W',\n", - " 'F200W']" + "Similarly to DS9, ImViz allows you to interactively view these images and the corresponding source catalog as well." ] }, { @@ -464,52 +512,39 @@ "outputs": [], "source": [ "imviz = Imviz()\n", - "for ii in range(len(image3_i2d)):\n", + "viewer = imviz.default_viewer\n", + "\n", + "for img in image3_i2d:\n", + " print(f'Plotting: {img}')\n", + " label = f\"obs{fits.getval(img, 'OBSERVTN')} {fits.getval(img, 'PUPIL')}\"\n", " with warnings.catch_warnings():\n", " warnings.simplefilter('ignore')\n", - " imviz.load_data(image3_i2d[ii], data_label=image3_label[ii])\n", - "imviz.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "98bcbaf8", - "metadata": {}, - "outputs": [], - "source": [ - "viewer = imviz.default_viewer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ad8d9c26", - "metadata": {}, - "outputs": [], - "source": [ - "linking = imviz.plugins['Links Control']\n", - "linking.link_type = 'WCS'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8369edea", - "metadata": {}, - "outputs": [], - "source": [ + " imviz.load_data(img, data_label=label)\n", + "\n", + " # this aligns the image to use the WCS coordinates\n", + " linking = imviz.plugins['Links Control']\n", + " linking.link_type = 'WCS'\n", + "\n", + " # also plot the associated catalog\n", + " cat = Table.read(img.replace('i2d.fits', 'cat.ecsv'))\n", + " # format the table into the format imviz expects\n", + " t_xy = Table({'x': cat['xcentroid'],\n", + " 'y': cat['ycentroid']})\n", + " viewer.marker = {'color': 'orange', 'alpha': 1, 'markersize': 20, 'fill': False}\n", + " viewer.add_markers(t_xy, marker_name=f\"{label} catalog\")\n", + "\n", + "# This changes the stretch of all of the images\n", "plotopt = imviz.plugins['Plot Options']\n", - "plotopt.multiselect = True\n", "plotopt.select_all(viewers=True, layers=True)\n", "plotopt.stretch_preset = '99.5%'\n", - "plotopt.multiselect = False" + " \n", + "imviz.show()" ] }, { "cell_type": "code", "execution_count": null, - "id": "235aa97d", + "id": "3201b7b6-64dd-4367-a702-1fa8f547eacb", "metadata": {}, "outputs": [], "source": [] @@ -531,7 +566,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.11.7" } }, "nbformat": 4, From b20d96f4937016a57841ae25c5cd9e9d12d3612a Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Wed, 31 Jan 2024 16:12:20 -0500 Subject: [PATCH 15/62] major additions and restructure of this notebook, including adding notebook 05 into this --- .../04_run_pipeline_imaging_level2and3.ipynb | 605 +++++++++++++++--- 1 file changed, 527 insertions(+), 78 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb index effa06c91..c42a18067 100644 --- a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb @@ -20,11 +20,14 @@ " - [Image2](#default_image2)\n", " - [Image3](#default_image3)\n", " - [Inspecting Default Results](#view_default)\n", - "- [Custom Imaging Pipeline Run]\n", - " - [Image3]\n", - " - [Inspecting Custom Results]\n", + "- [Custom Imaging Pipeline Run](#custom)\n", + " - [Image3](#custom_image3)\n", + " - [Inspecting Custom Results](#view_custom)\n", + "- [Refining the Source Catalog Further](#source_cat)\n", + " - [Matching Source IDs Across Catalogs](#match_sources) \n", + " - [Manually Editing the Source Catalog](#manual_cat)\n", "\n", - "**Author**: Camilla Pacifici (cpacifici@stsci.edu), Rachel Plesha (rplesha@stsci.edu), JWebbinar notebooks.
\n", + "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", "**Last modified**: January 2024" ] }, @@ -51,7 +54,9 @@ "import warnings\n", "import numpy as np\n", "import pandas as pd\n", + "import astropy.units as u\n", "from astropy.io import fits\n", + "from astropy.coordinates import SkyCoord\n", "from astropy.table import Table\n", "from matplotlib import pyplot as plt\n", "%matplotlib inline" @@ -137,6 +142,18 @@ " pass" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "b88f9b55-dbde-466c-9647-86e5fff3fdcc", + "metadata": {}, + "outputs": [], + "source": [ + "for temp_dir in [default_run_image3, custom_run_image3]:\n", + " if not os.path.exists(temp_dir):\n", + " os.mkdir(temp_dir)" + ] + }, { "cell_type": "markdown", "id": "10824c6a-4843-4f31-beb7-05ecc58e8b84", @@ -237,10 +254,19 @@ "cell_type": "code", "execution_count": null, "id": "dff099be", - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "for img2_asn in image2_asns:\n", + " # check if the calibrated file already exists\n", + " asn_data = json.load(open(img2_asn))\n", + " cal_file = f\"{asn_data['products'][0]['name']}_cal.fits\"\n", + " if os.path.exists(cal_file):\n", + " print(cal_file, 'cal file already exists.')\n", + " continue\n", + " # if not, calibrated with image2\n", " img2 = Image2Pipeline.call(img2_asn, save_results=True)" ] }, @@ -315,9 +341,17 @@ { "cell_type": "markdown", "id": "ca3bd322-7f67-43c4-8dae-dc47779462ee", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ - "Image3 is where we can make some adjustments to obtain a better output source catalog. The `cal.fits` files will be calibrated into a single combined `i2d.fits` image. More information about the steps performed in the Image3 part of the pipeline can be found in the [Image3 pipeline documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_image3.html)." + "Image3 is where we can make some adjustments to obtain a better output source catalog. The `cal.fits` files will be calibrated into a single combined `i2d.fits` image. More information about the steps performed in the Image3 part of the pipeline can be found in the [Image3 pipeline documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_image3.html).\n", + "\n", + "**Note: Image3 can take a while to run**" ] }, { @@ -328,46 +362,16 @@ "outputs": [], "source": [ "for img3_asn in image3_asns:\n", + " # check if the calibrated file already exists\n", + " asn_data = json.load(open(img3_asn))\n", + " cal_file = os.path.join(default_run_image3, f\"{asn_data['products'][0]['name']}_i2d.fits\")\n", + " if os.path.exists(cal_file):\n", + " print(cal_file, 'cal file already exists.')\n", + " continue\n", + " # if not, calibrated with image3\n", " img3 = Image3Pipeline.call(img3_asn, save_results=True, output_dir=default_run_image3)" ] }, - { - "cell_type": "markdown", - "id": "eaabda11", - "metadata": {}, - "source": [ - "I'll first run the step with the default options, check the catalog and rerun with adjusted parameters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3f2f7f0f", - "metadata": {}, - "outputs": [], - "source": [ - "for file in asn_image3_local:\n", - " print(file)\n", - " image3 = Image3Pipeline()\n", - " result = image3.call(file,\n", - " steps={\n", - " 'source_catalog':{'kernel_fwhm':5.0,\n", - " 'snr_threshold':10.0,\n", - " 'npixels':50,\n", - " 'deblend':True,\n", - " },\n", - " 'tweakreg':{'snr_threshold':20,\n", - " 'abs_refcat':'GAIADR2', #Try DR3\n", - " 'save_catalogs':True,\n", - " 'searchrad':3.0,\n", - " 'kernel_fwhm':2.302,\n", - " 'fitgeometry':'shift',\n", - " },\n", - " },\n", - " save_results=True,\n", - " output_dir=data_dir_out_image3)" - ] - }, { "cell_type": "markdown", "id": "33f0dbeb-4caf-4d97-8b2c-3255f906e76b", @@ -455,43 +459,71 @@ "metadata": {}, "outputs": [], "source": [ - "fig2 = plt.figure(figsize=(10, 20))\n", - "\n", - "cols = 2\n", - "rows = len(image3_i2d)\n", - "\n", - "for plt_num, img in enumerate(np.sort(np.concatenate([image3_segm, image3_i2d]))):\n", - "\n", - " # determine where the subplot should be\n", - " xpos = (plt_num%40) % cols\n", - " ypos = ((plt_num%40) // cols) # // to make it an int.\n", - "\n", - " # make the subplot\n", - " ax = plt.subplot2grid((rows, cols), (ypos, xpos))\n", + "# we will look at this multiple times, so let's define this as a function\n", + "def plot_image_and_segmentation_map(i2d_images, segm_images, xmin=1250, xmax=1750, ymin=1250, ymax=1750, cat_suffix='cat.ecsv'):\n", + " \n", + " cols = 2\n", + " rows = len(i2d_images)\n", "\n", - " # plot the image\n", - " with fits.open(img) as hdu:\n", - " ax.imshow(hdu[1].data, vmin=0, vmax=0.3, origin='lower')\n", - " title = f\"obs{hdu[0].header['OBSERVTN']} {hdu[0].header['PUPIL']}\"\n", + " fig = plt.figure(figsize=(10, 10*(rows/2)) )\n", "\n", - " # also plot the associated catalog\n", - " if 'i2d' in img:\n", - " cat = Table.read(img.replace('i2d.fits', 'cat.ecsv'))\n", - " ax.set_title(f\"{title} combined image\")\n", - " else:\n", - " ax.set_title(f\"{title} segmentation map\")\n", + " for plt_num, img in enumerate(np.sort(np.concatenate([segm_images, i2d_images]))):\n", + " \n", + " # determine where the subplot should be\n", + " xpos = (plt_num%40) % cols\n", + " ypos = ((plt_num%40) // cols) # // to make it an int.\n", "\n", - " extended_sources = cat[cat['is_extended'] == True]\n", - " point_sources = cat[cat['is_extended'] == False]\n", - " ax.scatter(extended_sources['xcentroid'], extended_sources['ycentroid'], s=20, facecolors='None', edgecolors='orangered', alpha=0.9)\n", - " ax.scatter(point_sources['xcentroid'], point_sources['ycentroid'], s=20, facecolors='None', edgecolors='dimgrey', alpha=0.9)\n", + " # make the subplot\n", + " ax = plt.subplot2grid((rows, cols), (ypos, xpos))\n", + " \n", + " if 'i2d' in img:\n", + " cat = Table.read(img.replace('i2d.fits', cat_suffix))\n", + " cmap = 'gist_gray'\n", + " else:\n", + " cmap = 'tab20c_r'\n", + " \n", + " # plot the image\n", + " with fits.open(img) as hdu:\n", + " ax.imshow(hdu[1].data, vmin=0, vmax=0.3, origin='lower', cmap=cmap)\n", + " title = f\"obs{hdu[0].header['OBSERVTN']} {hdu[0].header['PUPIL']}\"\n", + " \n", + " # also plot the associated catalog\n", + " extended_sources = cat[cat['is_extended'] == True]\n", + " point_sources = cat[cat['is_extended'] == False]\n", + " \n", + " for color, sources in zip(['darkorange', 'black'], [extended_sources, point_sources]):\n", + " # plotting the sources\n", + " ax.scatter(sources['xcentroid'], sources['ycentroid'], s=20, facecolors='None', edgecolors=color, alpha=0.9)\n", " \n", - " # zooming in on a smaller region\n", - " ax.set_xlim(1250, 1750)\n", - " ax.set_ylim(1250, 1750)\n", + " # adding source labels \n", + " for i, source_num in enumerate(sources['label']):\n", + " ax.annotate(source_num, \n", + " (sources['xcentroid'][i]+0.5, sources['ycentroid'][i]+0.5), \n", + " fontsize=8,\n", + " color=color)\n", + " if 'i2d' in img:\n", + " ax.set_title(f\"{title} combined image\")\n", + " else:\n", + " ax.set_title(f\"{title} segmentation map\")\n", + " \n", + " # zooming in on a smaller region\n", + " ax.set_xlim(xmin, xmax)\n", + " ax.set_ylim(ymin, ymax)\n", + " \n", + " # Helps to make the axes not overlap ; you can also set this manually if this doesn't work\n", + " plt.tight_layout()\n", "\n", - "# Helps to make the axes not overlap ; you can also set this manually if this doesn't work\n", - "plt.tight_layout()" + " return fig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0599d52e-7a92-4c16-92e7-59ae62cfe8d9", + "metadata": {}, + "outputs": [], + "source": [ + "default_fig = plot_image_and_segmentation_map(image3_i2d, image3_segm)" ] }, { @@ -541,13 +573,430 @@ "imviz.show()" ] }, + { + "cell_type": "markdown", + "id": "4ea23a1b-4b6f-4ee1-9e2b-ff132dd9127d", + "metadata": {}, + "source": [ + "\n", + "## Custom Imaging Pipeline Run" + ] + }, + { + "cell_type": "markdown", + "id": "14216c26-cf89-4f15-b7ec-b5a8ca0071ba", + "metadata": {}, + "source": [ + "\n", + "### Image3" + ] + }, + { + "cell_type": "markdown", + "id": "ccc697e5-92de-4a2a-a9e1-3f81d20fcf27", + "metadata": {}, + "source": [ + "Try editing a few parameters and compare the outcomes to the default run above, at first for a single file.\n", + "\n", + "When we call the image3 pipeline, we can add modifications to a specific step of the pipeline. In this case we're going to edit the `source_catalog` and `tweakreg` steps of the pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f2f7f0f", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "image3_asns = np.sort(glob.glob('*image3*asn*.json'))\n", + "test_asn = image3_asns[1]\n", + "\n", + "# check if the calibrated file already exists\n", + "asn_data = json.load(open(test_asn))\n", + "i2d_file = os.path.join(custom_run_image3, f\"{asn_data['products'][0]['name']}_i2d.fits\")\n", + "\n", + "if os.path.exists(i2d_file):\n", + " print(cal_file, 'cal file already exists.')\n", + "else:\n", + " # call the image3 pipeline in the same way as before, but add a few new modifications\n", + " cust_img3 = Image3Pipeline.call(test_asn,\n", + " steps={\n", + " 'source_catalog':{'kernel_fwhm':5.0,\n", + " 'snr_threshold':10.0,\n", + " 'npixels':50,\n", + " 'deblend':True,\n", + " },\n", + " 'tweakreg':{'snr_threshold':20,\n", + " 'abs_refcat':'GAIADR3',\n", + " 'save_catalogs':True,\n", + " 'searchrad':3.0,\n", + " 'kernel_fwhm':2.302,\n", + " 'fitgeometry':'shift',\n", + " },\n", + " },\n", + " save_results=True,\n", + " output_dir=custom_run_image3)" + ] + }, + { + "cell_type": "markdown", + "id": "17d4cdae-0cad-4385-b93b-6976ecf90898", + "metadata": {}, + "source": [ + "\n", + "### Inspecting Custom Results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96e15c74-65e5-45b2-bf83-4e512cff4c6c", + "metadata": {}, + "outputs": [], + "source": [ + "default_i2d = os.path.join(default_run_image3, os.path.basename(i2d_file))\n", + "compare_i2ds = [i2d_file, default_i2d]\n", + "compare_segm = [i2d_file.replace('i2d.fits', 'segm.fits'), default_i2d.replace('i2d.fits', 'segm.fits')]\n", + "\n", + "compare_fig = plot_image_and_segmentation_map(compare_i2ds, compare_segm)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ac548eb-ae6a-4c08-ba87-8d5714a9880b", + "metadata": {}, + "outputs": [], + "source": [ + "imviz = Imviz()\n", + "viewer = imviz.default_viewer\n", + "\n", + "for img, label in zip([i2d_file, os.path.join(default_run_image3, os.path.basename(i2d_file))], ['custom', 'default']):\n", + " print(f'Plotting: {img}')\n", + " title = f\"{label} obs{fits.getval(img, 'OBSERVTN')} {fits.getval(img, 'PUPIL')}\"\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter('ignore')\n", + " imviz.load_data(img, data_label=title)\n", + "\n", + " # this aligns the image to use the WCS coordinates\n", + " linking = imviz.plugins['Links Control']\n", + " linking.link_type = 'WCS'\n", + "\n", + " # also plot the associated catalog\n", + " cat = Table.read(img.replace('i2d.fits', 'cat.ecsv'))\n", + " # format the table into the format imviz expects\n", + " t_xy = Table({'x': cat['xcentroid'],\n", + " 'y': cat['ycentroid']})\n", + " viewer.marker = {'color': 'orange', 'alpha': 1, 'markersize': 20, 'fill': False}\n", + " viewer.add_markers(t_xy, marker_name=f\"{label} catalog\")\n", + "\n", + "# This changes the stretch of all of the images\n", + "plotopt = imviz.plugins['Plot Options']\n", + "plotopt.select_all(viewers=True, layers=True)\n", + "plotopt.stretch_preset = '99.5%'\n", + " \n", + "imviz.show()" + ] + }, + { + "cell_type": "markdown", + "id": "48ae7dc1-45ce-41a6-a5a3-c07226efc912", + "metadata": {}, + "source": [ + "Calibrate the remaining images if you are happy with the above results" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "3201b7b6-64dd-4367-a702-1fa8f547eacb", + "id": "4787d1bb-57a7-4056-b93e-c5e7497d14cf", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "image3_asns = np.sort(glob.glob('*image3*asn*.json'))\n", + "\n", + "for img3_asn in image3_asns:\n", + " # check if the calibrated file already exists\n", + " asn_data = json.load(open(img3_asn))\n", + " i2d_file = os.path.join(custom_run_image3, f\"{asn_data['products'][0]['name']}_i2d.fits\")\n", + " if os.path.exists(i2d_file):\n", + " print(i2d_file, 'cal file already exists.')\n", + " continue\n", + " # call the image3 pipeline in the same way as before, but add a few new modifications\n", + " cust_img3 = Image3Pipeline.call(img3_asn,\n", + " steps={\n", + " 'source_catalog':{'kernel_fwhm':5.0,\n", + " 'snr_threshold':10.0,\n", + " 'npixels':50,\n", + " 'deblend':True,\n", + " },\n", + " 'tweakreg':{'snr_threshold':20,\n", + " 'abs_refcat':'GAIADR3',\n", + " 'save_catalogs':True,\n", + " 'searchrad':3.0,\n", + " 'kernel_fwhm':2.302,\n", + " 'fitgeometry':'shift',\n", + " },\n", + " },\n", + " save_results=True,\n", + " output_dir=custom_run_image3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8286e568-a553-4434-93ff-74829b79d110", + "metadata": {}, + "outputs": [], + "source": [ + "# These are all resuts from the Image3 pipeline\n", + "cust_image3_i2d = np.sort(glob.glob(os.path.join(custom_run_image3, '*i2d.fits'))) # combined image over multiple dithers/mosaic\n", + "cust_image3_segm = np.sort(glob.glob(os.path.join(custom_run_image3, '*segm.fits'))) # segmentation map that defines the extent of a source\n", + "\n", + "custom_fig = plot_image_and_segmentation_map(cust_image3_i2d, cust_image3_segm)" + ] + }, + { + "cell_type": "markdown", + "id": "812a461f-27f5-4ae7-84bd-4c4cafe0c603", + "metadata": {}, + "source": [ + "\n", + "## Refining the Source Catalog Further" + ] + }, + { + "cell_type": "markdown", + "id": "191a12fb-0989-46d2-8e62-792cf3b23f4a", + "metadata": {}, + "source": [ + "In the above cases, we have modified the results using the pipeline directly. It might be the case that perhaps you want to use someone else's custom source catalog, or modify the source catalog even further from what was output by the pipeline. In these cases, we will then need to modify the spec2 ASN files to point to the new source catalog, which will be discussed in the spec2 notebook. Additionally, it can be useful to match all of the source IDs across the different catalogs. In this case, there are four different catalogs created by the pipeline that identify the same RA/Dec or X/Y pixel location as a different source ID, so we will edit those to have the same source ID values across the catalogs. These extra steps aren't always necessary, but could be helpful in analyses of NIRISS WFSS data." + ] + }, + { + "cell_type": "markdown", + "id": "4eef927a-4301-481c-b730-d80219b9d14e", + "metadata": {}, + "source": [ + "\n", + "#### Matching Source IDs Across Catalogs\n", + "\n", + "In the above figures, you can see that the same source has multiple source IDs. Here we want to match all of the source IDs across all observations to be sure we are talking about the same source regardless of which filter or observation we look at. To do this, we use the astropy `match_to_catalog_3d` function and rebase the labels." + ] + }, + { + "cell_type": "markdown", + "id": "6ca9e9eb-6420-421a-818b-1faab1db788a", + "metadata": {}, + "source": [ + "The first step is to decide on a base catalog to match all of the other catalogs to. Here we'll use the the observation 001 F115W filter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d024dff0-6ffc-408c-ae86-b65f766e73ef", + "metadata": {}, + "outputs": [], + "source": [ + "custom_cats = np.sort(glob.glob(os.path.join(custom_run_image3, '*niriss_clear-f????_cat.ecsv'))) # cat filename format from image3 results\n", + "print(\"All image3 catalogs:\\n\", custom_cats)\n", + "\n", + "base_cat = Table.read(custom_cats[0])\n", + "\n", + "# save out the base catalog with a new name to be consistent\n", + "base_cat_name = custom_cats[0].replace('cat.ecsv', 'source-match_cat.ecsv')\n", + "base_cat.write(base_cat_name, overwrite=True)\n", + "print(\"\\nBase catalog:\", base_cat_name)" + ] + }, + { + "cell_type": "markdown", + "id": "589f1d52-3915-43c6-93b7-32915139e7d6", + "metadata": {}, + "source": [ + "Loop through the remaining catalogs to match the IDs based off of sky coordinates matching to within 1 arcsecond." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55f6ce9b-d995-489c-8f1b-b6ce76e4b5ee", + "metadata": {}, + "outputs": [], + "source": [ + "max_sep = 1 * u.arcsec # adjust if necessary\n", + "\n", + "base_sky = base_cat['sky_centroid']\n", + "\n", + "for to_match_cat in custom_cats[1:]:\n", + " # read in the catalog\n", + " other_cat = Table.read(to_match_cat)\n", + " other_sky = other_cat['sky_centroid']\n", + "\n", + " # find the matching sources between the two catalogs based on sky coordinates\n", + " idx, d2d, d3d = base_sky.match_to_catalog_3d(other_sky)\n", + " sep_constraint = d2d < max_sep\n", + " base_matches = base_cat[sep_constraint]\n", + " other_matches = other_cat[idx[sep_constraint]]\n", + "\n", + " # rebase the ID values to be the same\n", + " other_matches['label'] = base_matches['label']\n", + "\n", + " # save out the new catalog\n", + " match_cat_name = to_match_cat.replace('cat.ecsv', 'source-match_cat.ecsv')\n", + " other_matches.write(match_cat_name, overwrite=True)\n", + " print('Saved:', match_cat_name)" + ] + }, + { + "cell_type": "markdown", + "id": "f080887a-c1dd-435e-ae8a-72dcfebf9606", + "metadata": {}, + "source": [ + "Look at the new source label numbers. They should match!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e0e4837-f956-4cd2-b4f9-68a60d647605", + "metadata": {}, + "outputs": [], + "source": [ + "new_cat_fig = plot_image_and_segmentation_map(cust_image3_i2d, cust_image3_segm, cat_suffix='source-match_cat.ecsv')" + ] + }, + { + "cell_type": "markdown", + "id": "686b13dc-0573-45ec-8828-9be428bf32ea", + "metadata": {}, + "source": [ + "\n", + "#### Manually Editing the Source Catalog\n", + "\n", + "Looking ahead to the WFSS extraction, it might be that we only really care about one or two sources at any particular moment. In this case, it's helpful to pair down the source catalog to just that source to make it easier to look at the extracted 1-D spectrum in the WFSS data.\n", + "\n", + "- [Source Catalog Column Information](https://jwst-pipeline.readthedocs.io/en/latest/jwst/source_catalog/main.html#output-products)" + ] + }, + { + "cell_type": "markdown", + "id": "a379ccea-58a4-4440-b1f5-26c9736e36c6", + "metadata": {}, + "source": [ + "To start, look at the current custom source catalog for one of the filters to get an idea of what is contained in the catalogs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e51fa992-9785-4c96-a4fa-99572e16f63c", + "metadata": {}, + "outputs": [], + "source": [ + "cat_name = np.sort(glob.glob(os.path.join(custom_run_image3, '*source-match_cat.ecsv')))[3]\n", + "print(cat_name)\n", + "cat = Table.read(cat_name)\n", + "cat[['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']]" + ] + }, + { + "cell_type": "markdown", + "id": "66930bfc-8647-4e00-8369-936eaff148a9", + "metadata": {}, + "source": [ + "There may be multiple ways to look for a source, so shown below are three options:\n", + "1. With a known RA/Dec of an object\n", + "2. A known x/y location of an object\n", + "3. With a source ID of an object. Note that we are using the rebased source catalogs here for the IDs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a5af10e-90c8-4e08-bff7-b6b798f9bea5", + "metadata": {}, + "outputs": [], + "source": [ + "# with a known RA/Dec\n", + "desired_ra = 53.16237 \n", + "desired_dec = -27.775\n", + "\n", + "c = SkyCoord(ra=desired_ra*u.degree, dec=desired_dec*u.degree)\n", + "nearest_id, distance_2d, distance_3d = c.match_to_catalog_sky(cat['sky_centroid']) \n", + "\n", + "cat[['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']][nearest_id]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04bd4079-c774-471d-94da-004a7fd287e8", + "metadata": {}, + "outputs": [], + "source": [ + "# alternatively with a known X/Y pixel location\n", + "known_x = 1580\n", + "known_y = 1400\n", + "\n", + "nearest_pos = [np.sqrt((x-known_x)**2 + (y-known_y)**2) for x, y in zip(cat['xcentroid'], cat['ycentroid'])]\n", + "\n", + "wh_nearest = np.where(np.array(nearest_pos) == min(nearest_pos))[0][0]\n", + "\n", + "cat[['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']][wh_nearest]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "572fe4a4-3d4d-4364-9672-7b2d23154fad", + "metadata": {}, + "outputs": [], + "source": [ + "# alternatively with a known source number\n", + "source = 155\n", + "\n", + "wh_source = np.where(np.array(cat['label'] == source))[0][0]\n", + "\n", + "cat[['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']][wh_source]" + ] + }, + { + "cell_type": "markdown", + "id": "3303d1c5-d484-46d1-9472-e57862660c31", + "metadata": {}, + "source": [ + "Using any of the three methods above, write out the new catalog.\n", + "- with RA/Dec: `new_cat = Table(cat[nearest_id])`\n", + "- with x/y: `new_cat = Table(cat[wh_nearest])`\n", + "- with source ID: `new_cat = Table(cat[wh_source])`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aae96ac3-a587-4672-8edf-8bb327eea6e5", + "metadata": {}, + "outputs": [], + "source": [ + "new_cat = Table(cat[wh_source]) # turn the row instance into a dataframe again\n", + "\n", + "# save the new catalog with a unique name\n", + "new_cat_name = cat_name.replace('cat.ecsv', f'source{source}_cat.ecsv')\n", + "new_cat.write(new_cat_name, overwrite=True)\n", + "print('Saved:', new_cat_name)" + ] + }, + { + "cell_type": "markdown", + "id": "9d2a8566-eb1c-4df0-b4eb-f24c6edc8e56", + "metadata": {}, + "source": [ + "Once we have an updated source catalog that we are content with, we can move on to the spec2 step of the pipeline. It likely will be necessary to come back to this step after running spec2." + ] } ], "metadata": { From 3a0e60c92006c769771d883318feefac808d5316 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 11:01:19 -0500 Subject: [PATCH 16/62] Moving the creation of the info csv here & removing the file structure bit of code --- .../00_mast_query_progID.ipynb | 101 +++++++++--------- 1 file changed, 53 insertions(+), 48 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb index 533b20e9b..4e8bc007d 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb @@ -10,7 +10,7 @@ "\n", "**Use case**: use MAST to download data products.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: astropy, astroquery, numpy, os, glob, (yaml)
\n", + "**Tools**: astropy, astroquery, glob, numpy, os, pandas, (yaml)
\n", "**Cross-instrument**: all
\n", "\n", "**Content**\n", @@ -21,11 +21,11 @@ "- [Filter and Download Products](#filter)\n", " - [Filtering Data Before Downloading](#filter_data)\n", " - [Downloading Data](#downloading)\n", - "- [Reorganize Directory Structure](#reorg)\n", + "- [Create a Keyword Dataframe](#keyword)\n", "\n", "\n", - "**Author**: Camilla Pacifici (cpacifici@stsci.edu) & Rachel Plesha (rplesha@stsci.edu)
\n", - "**Last modified**: January 2024\n", + "**Author**: Camilla Pacifici (cpacifici@stsci.edu) & Rachel Plesha (rplesha@stsci.edu) & Jo Taylor (jotaylor@stsci.edu)
\n", + "**Last modified**: February 2024\n", "\n", "This notebook was inspired by the [JWebbinar session about MAST](https://github.com/spacetelescope/jwebbinar_prep/blob/main/mast_session/Crowded_Field/Crowded_Field.ipynb)." ] @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "7e92dee7", "metadata": {}, "outputs": [], @@ -52,7 +52,8 @@ "from astroquery.mast import Observations\n", "import numpy as np\n", "import os\n", - "import glob" + "import glob\n", + "import pandas as pd" ] }, { @@ -347,7 +348,7 @@ " print('Products downloaded:\\n', filtered_products['productFilename'])\n", " \n", " # only downloading the first batch of 5 observations\n", - " break # comment this out if you want to download everything" + " #break # comment this out if you want to download everything" ] }, { @@ -363,60 +364,64 @@ }, { "cell_type": "markdown", - "id": "a1eb6450", + "id": "e840d7c4-3643-4152-bd04-bfdcc150cb3e", "metadata": {}, "source": [ - "\n", - "## Reorganize Directory Structure\n", + "\n", + "## Create a Keyword Dataframe\n", "\n", - "This section takes the downloaded data and sorts it into a slightly different file structure to work with later in the notebooks. The expected format for this section is that all downloaded data are in a single directory (`flat=True`). The new file stucture is as follows:\n", - "- all rate files under NGDEEP/rate\n", - "- level 2 association files under NGDEEP/asn_level2\n", - "- level 3 association files under NGDEEP/asn_level3" + "The purpose of this function is to have a better idea of what data are available to you. Additionally, you will be able to use this dataframe to select specific files that match the mode you would like to take a closer look at." ] }, { "cell_type": "code", "execution_count": null, - "id": "5688a317", + "id": "949795c1-5ab9-45a5-aaf2-ba30593a1505", "metadata": {}, "outputs": [], "source": [ - "# first, make all of the new directories\n", - "topdir = 'data/NGDEEP'\n", - "if not os.path.exists(topdir):\n", - " os.mkdir(topdir)\n", - " print('Created:', topdir)\n", + "ratefile_datadir = 'data/'\n", "\n", - "for subdir in ['rate', 'asn_level2', 'asn_level3']:\n", - " new_subdir = os.path.join(topdir, subdir)\n", - " if not os.path.exists(new_subdir):\n", - " os.mkdir(new_subdir)\n", - " print('Created:', new_subdir)\n", + "# first look for all of the rate files you have downloaded\n", + "rate_files = glob.glob(os.path.join(ratefile_datadir, \"*rate.fits\"))\n", "\n", - "file_dict = {'rate' : 'rate',\n", - " 'image2' : 'asn_level2',\n", - " 'spec2' : 'asn_level2',\n", - " 'image3' : 'asn_level3',\n", - " 'spec3' : 'asn_level3',\n", - " }\n", + "for file_num, ratefile in enumerate(rate_files):\n", "\n", - "# now move all of the files to the appropriate locations\n", - "for filename in glob.glob('data/*.fits') + glob.glob('data/*.json'):\n", - " try:\n", - " index = filename.split('_')[2] # json files; looking for image2/image3 or spec2/spec3\n", - " subdir = file_dict[index]\n", - " except KeyError:\n", - " try:\n", - " index2 = filename.split('_')[-1].split('.')[0] # rate files\n", - " subdir = file_dict[index2]\n", - " except KeyError:\n", - " print(f'Unrecognized index: {index} or {index2}')\n", - " continue\n", - " \n", - " new_file = os.path.join(topdir, subdir, os.path.basename(filename))\n", - " os.rename(filename, new_file)\n", - " print(f'Moved: {filename} to {new_file}')" + " rate_hdr = fits.getheader(ratefile) # Primary header for each rate file\n", + "\n", + " # information we want to store that might be useful to us later for evaluating the data\n", + " temp_hdr_dict = {\"FILENAME\" : ratefile,\n", + " \"TARG_RA\" : [rate_hdr[\"TARG_RA\"]],\n", + " \"TARG_DEC\" : [rate_hdr[\"TARG_DEC\"]],\n", + " \"FILTER\" : [rate_hdr[\"FILTER\"]], # Grism; GR150R/GR150C\n", + " \"PUPIL\" : [rate_hdr[\"PUPIL\"]], # Filter used; F090W, F115W, F140M, F150W F158M, F200W\n", + " \"EXPSTART\" : [rate_hdr['EXPSTART']], # Exposure start time (MJD)\n", + " \"PATT_NUM\" : [rate_hdr[\"PATT_NUM\"]], # Position number within dither pattern for WFSS\n", + " \"NUMDTHPT\" : [rate_hdr[\"NUMDTHPT\"]], # Total number of points in entire dither pattern\n", + " \"XOFFSET\" : [rate_hdr[\"XOFFSET\"]], # X offset from pattern starting position for NIRISS (arcsec)\n", + " \"YOFFSET\" : [rate_hdr[\"YOFFSET\"]] # Y offset from pattern starting position for NIRISS (arcsec)\n", + " }\n", + "\n", + " # Turn the dictionary into a pandas dataframe\n", + " if file_num == 0:\n", + " # if this is the first file, make an initial dataframe\n", + " rate_df = pd.DataFrame(temp_hdr_dict)\n", + " else:\n", + " # otherwise, append to the dataframe for each file\n", + " new_data_df = pd.DataFrame(temp_hdr_dict)\n", + "\n", + " # merge the two dataframes together to create a dataframe with all \n", + " rate_df = pd.concat([rate_df, new_data_df], ignore_index=True, axis=0)\n", + "\n", + "rate_dfsort = rate_df.sort_values('EXPSTART', ignore_index=False)\n", + "\n", + "# Save the dataframe to a file to read in later, if desired\n", + "outfile = './list_ngdeep_rate.csv'\n", + "rate_dfsort.to_csv(outfile, sep=',', index=False)\n", + "print('Saved:', outfile)\n", + "\n", + "# Look at the resulting dataframe\n", + "rate_df" ] }, { @@ -444,7 +449,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.11.7" } }, "nbformat": 4, From 30fe14563c807b24f43118ddf6c904a23fba80e5 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 11:42:52 -0500 Subject: [PATCH 17/62] changing notebook to only be running individual steps --- .../03_imviz_level2.ipynb | 338 +++++++----------- 1 file changed, 127 insertions(+), 211 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb b/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb index 881aae2c8..573fa1ba5 100644 --- a/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb @@ -5,348 +5,264 @@ "id": "28608b44", "metadata": {}, "source": [ - "# Check flat fielded images from NGDEEP\n", + "# Running Individual Pipeline Steps\n", "\n", - "This is to look at the images and see if there are problems." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "216f4a35", - "metadata": {}, - "outputs": [], - "source": [ - "from jdaviz import Imviz\n", - "import warnings\n", - "from astropy.io import ascii, fits\n", - "from astropy.table import Table\n", - "from astropy.coordinates import SkyCoord, Angle\n", - "import astropy.units as u\n", - "import numpy as np\n", - "import os\n", - "import pandas as pd\n", - "import glob" + "This notebook walks through calibrating the data with individual pipeline steps rather than running the entire pipeline stage.\n", + "\n", + "**Use case**: When using a package outside of the standard JWST pipeline, there may be certain steps that are still helpful to utilize within the JWST pipeline.
\n", + "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", + "**Tools**: astropy, glob, jdaviz, jwst, matplotlib, numpy, os, pandas, warnings
\n", + "**Cross-instrument**: NIRISS
\n", + "\n", + "**Content**\n", + "- [Imports & Data Setup](#imports)\n", + "- [Running Individual Pipeline Steps](#pipeline_steps)\n", + " - [Assign WCS Step](#wcs_step)\n", + " - [Flat Field Step](#ff_step)\n", + " - [Compare Rate vs. Flat Fielded Data](#compare)\n", + "\n", + "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu)
\n", + "**Last modified**: February 2024" ] }, { "cell_type": "markdown", - "id": "d4c0f2ce", + "id": "5ebb20f2-fb41-42c6-96a6-b35dfe77bab7", "metadata": {}, "source": [ - "## Read list of files" + "\n", + "## Imports & Data Setup" ] }, { "cell_type": "code", "execution_count": null, - "id": "05afdbfc", + "id": "216f4a35", "metadata": {}, "outputs": [], "source": [ - "def write_keywords_csv(datadir):\n", - " # set up an empty dataframe initially to fill in\n", - " df = pd.DataFrame(columns=[\"FILENAME\",\n", - " \"TARG_RA\", \n", - " \"TARG_DEC\", \n", - " \"FILTER\", # Grism; GR150R/GR150C\n", - " \"PUPIL\", # Filter used; F090W, F115W, F140M, F150W F158M, F200W\n", - " \"PATT_NUM\", # Position number within dither pattern for WFSS\n", - " \"NUMDTHPT\", # Total number of points in entire dither pattern\n", - " \"XOFFSET\", # X offset from pattern starting position for NIRISS (arcsec)\n", - " \"YOFFSET\"]) # Y offset from pattern starting position for NIRISS (arcsec)\n", - "\n", - " for ratefile in glob.glob(os.path.join(datadir, \"*rate.fits\")):\n", - " image = fits.open(ratefile)\n", - "\n", - " # create a new dataframe for each file\n", - " df2 = pd.DataFrame({\"FILENAME\" : ratefile,\n", - " \"TARG_RA\" : [image[0].header[\"TARG_RA\"]],\n", - " \"TARG_DEC\" : [image[0].header[\"TARG_DEC\"]],\n", - " \"FILTER\" : [image[0].header[\"FILTER\"]],\n", - " \"PUPIL\" : [image[0].header[\"PUPIL\"]],\n", - " \"PATT_NUM\" : [image[0].header[\"PATT_NUM\"]],\n", - " \"NUMDTHPT\" : [image[0].header[\"NUMDTHPT\"]],\n", - " \"XOFFSET\" : [image[0].header[\"XOFFSET\"]],\n", - " \"YOFFSET\" : [image[0].header[\"YOFFSET\"]]\n", - " })\n", - "\n", - " # merge the two dataframes together to create a dataframe with all \n", - " df = pd.concat([df, df2], ignore_index=True, axis=0)\n", - "\n", - " dfsort = df.sort_values('FILENAME', ignore_index=False)\n", + "import os\n", + "import glob\n", + "import warnings\n", + "import numpy as np\n", + "import pandas as pd\n", "\n", - " dfsort.to_csv(\"./list_ngdeep_rate.csv\", sep=',', index=False)\n", - " return dfsort" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bf65d59c", - "metadata": {}, - "outputs": [], - "source": [ - "listrate_file = './list_ngdeep_rate.csv'\n", + "import astropy.units as u\n", + "from astropy.io import ascii, fits\n", + "from astropy.table import Table\n", + "from astropy.coordinates import SkyCoord, Angle\n", "\n", - "if os.path.exists(listrate_file):\n", - " rate_df = Table.read(listrate_file)\n", - "else:\n", - " rate_df = write_keywords_csv('data/NGDEEP/rate')\n", + "from jdaviz import Imviz\n", + "from matplotlib import pyplot as plt\n", + "%matplotlib inline\n", "\n", - "rate_df[-1::]" + "import jwst\n", + "from jwst.assign_wcs import AssignWcsStep\n", + "from jwst.flatfield import FlatFieldStep" ] }, { - "cell_type": "markdown", - "id": "4d7f78b7", + "cell_type": "raw", + "id": "f0e5031b-f273-47d6-97b6-41e0fd2d3a47", "metadata": {}, "source": [ - "While you could look at the rate images, instead consider running the files through flatfield step of the pipeline to clean up detector artifacts.\n", - "\n", - "** Insert notebook 02 here as an optional function and call it. Fix the function to use call instead while doing this, too ** " + "# Update the CRDS path to your local directory\n", + "%env CRDS_PATH= (/grp/crds/jwst/)\n", + "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" ] }, { "cell_type": "markdown", - "id": "3a7a85e7", + "id": "3c208323-56f4-4fb5-b8c0-750b65b525aa", "metadata": {}, "source": [ - "## Check astrometry on direct images" + "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available, check here" ] }, { "cell_type": "code", "execution_count": null, - "id": "7e26b8a3", + "id": "b649638b-0353-4b4a-874d-949fc6610876", "metadata": {}, "outputs": [], "source": [ - "direct_image_df = rate_df[rate_df['FILTER'] == 'CLEAR'] # selecting out only the direct images\n", - "clearim = direct_image_df['FILENAME']\n", - "# optional flatfield step would be:\n", - "flatfield_im = [f.replace('rate.fits', 'flatfieldstep.fits') for f in clearim]\n", - "datalabel = direct_image_df['PUPIL']" + "print('jwst:', jwst.__version__)" ] }, { "cell_type": "markdown", - "id": "9144f386", + "id": "be56a062-bec7-4aff-922a-eaec53bdb346", "metadata": {}, "source": [ - "### Use Imviz batch load since it is many images\n", - "It takes about 1 minute to load all the images.\n", - "\n", - "** I need some basic instructions here to know how to navigate **" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "efbedea9", - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "imviz = Imviz()\n", - "for img, filt_label in zip(clearim, datalabel):\n", - " with warnings.catch_warnings():\n", - " warnings.simplefilter('ignore')\n", - " imviz.load_data(img, data_label=filt_label)\n", - "imviz.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c6ebfa64", - "metadata": {}, - "outputs": [], - "source": [ - "viewer = imviz.default_viewer" + "#### Data setup\n", + "Open the file that contains all of the rate files, and create an output directory for the calibrated files if it does not already exist" ] }, { "cell_type": "code", "execution_count": null, - "id": "6ed6c0c4", + "id": "3b9e37a5-d867-4047-9e22-ceb42a0e8a61", "metadata": {}, "outputs": [], "source": [ - "linking = imviz.plugins['Links Control']\n", - "linking.link_type = 'WCS'" + "# From the csv file we created earlier, find a list of all of the grism observations we will want to calibrate with spec2\n", + "listrate_file = 'list_ngdeep_rate.csv'\n", + "rate_df = pd.read_csv(listrate_file)" ] }, { "cell_type": "code", "execution_count": null, - "id": "53b3db0f", + "id": "48baec42-7aee-4731-8048-1054c9a127f4", "metadata": {}, "outputs": [], "source": [ - "# This also takes a bit of time (~30 seconds) to loop through all the images\n", - "plotopt = imviz.plugins['Plot Options']\n", - "plotopt.multiselect = True\n", - "plotopt.select_all(viewers=True, layers=True)\n", - "plotopt.stretch_preset = '99.5%'\n", - "plotopt.multiselect = False\n", - "#plotopt.stretch_function = 'Arcsinh'\n", - "#plotopt.stretch_vmin = -0.002\n", - "#plotopt.stretch_vmax = 0.16" + "all_rate_files = rate_df['FILENAME']\n", + "data_dir_out = 'data/calibrated_steps/' # directory where to save the calibrate files\n", + "\n", + "# if the directory does not exist that you want to save out to, make that directory first\n", + "if not os.path.exists(data_dir_out):\n", + " os.makedirs(data_dir_out)" ] }, { "cell_type": "markdown", - "id": "5a2a5552", + "id": "d9bfb726-26b5-46c1-99bf-3c86e0c9e3bb", "metadata": {}, "source": [ - "### Blink\n", - "Using the \"b\" key when on the viewer." + "\n", + "## Running Individual Pipeline Steps" ] }, { "cell_type": "markdown", - "id": "54cebc13", + "id": "da194621-1d85-4481-922a-118ec1f859ec", "metadata": {}, "source": [ - "**Data note**: there is a problem with the astrometry. The F200W filter does not match the others. I am not sure if F200W is the problem or the other two are the problem. Fixing the astrometry is out of scope for this notebook." + "While you could look at the rate images, instead consider running the files through the `assign_wcs` and `flat_field` steps of the pipeline to clean up detector artifacts." ] }, { "cell_type": "markdown", - "id": "015c5f3c", + "id": "0c83aab3-75de-4a9b-be3c-28669b4c0044", "metadata": {}, "source": [ - "## Check direct images and their corresponding WFSS images\n", + "\n", + "#### Assign WCS Step\n", "\n", - "Look at direct and dispersed images on the same dither position." + "The `assign_wcs` step of the pipeline is a critical part to obtaining the correct spectral trace cutouts for WFSS images. To read more about the step, visit the [AssignWCS jdox page](https://jwst-pipeline.readthedocs.io/en/latest/jwst/assign_wcs/main.html)." ] }, { "cell_type": "code", "execution_count": null, - "id": "fa07c90d", - "metadata": {}, + "id": "66548075-1cf6-4d58-80ef-833f46c021a4", + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "# Using the XOFFSET and YOFFSET keywords, match the direct image and dispersed (grism) images to each other\n", - "\n", - "file_dict = {}\n", - "for xoff, yoff, imgname in zip(direct_image_df['XOFFSET'], direct_image_df['YOFFSET'], \n", - " direct_image_df['FILENAME']):\n", - " match_pointing_df = rate_df[(rate_df['XOFFSET'] == xoff) & \n", - " (rate_df['YOFFSET'] == yoff) &\n", - " (rate_df['FILENAME'] != imgname)]\n", - " \n", - " # If there aren't any matches, keep moving\n", - " if not len(match_pointing_df):\n", - " continue\n", - " \n", - " # fill in the direct / dispersed dataframe for each of the filters\n", - " temp_filter = match_pointing_df.iloc[0]['PUPIL']\n", - " if temp_filter in list(file_dict.keys()):\n", - " # For now, only grab the first instance of the image/grism pair\n", - " continue\n", - " else:\n", - " print('**', temp_filter, imgname)\n", - " print(match_pointing_df.iloc[0]['FILENAME'])\n", - " print(match_pointing_df.iloc[0]['FILTER'])\n", - " file_dict[temp_filter] = {'direct_img' : imgname,\n", - " 'dispersed_img' : match_pointing_df.iloc[0]['FILENAME']}\n", - " print()" + "# Run assign_wcs\n", + "for ratefile in all_rate_files:\n", + " result = AssignWcsStep.call(ratefile, output_dir=data_dir_out, save_results=True)" ] }, { "cell_type": "code", "execution_count": null, - "id": "963c1f6a", + "id": "9f97a814-64e6-44a9-bd35-4db7cb1977a7", "metadata": {}, "outputs": [], "source": [ - "imviz2 = Imviz()\n", - "with warnings.catch_warnings():\n", - " warnings.simplefilter('ignore')\n", - " for filt, img_dict in file_dict.items(): # loop over all filters\n", - " for imgtype, filename in img_dict.items(): # loop over all files\n", - " imviz2.load_data(filename, data_label=f\"{filt} {imgtype}\")\n", - "imviz2.show()" + "# A quick sanity check to ensure that the files were calibrated.\n", + "# if this is zero, check the log message above for any errors that may have occurred during the calibration\n", + "wcsstep_files = glob.glob(os.path.join(data_dir_out, '*assignwcsstep*'))\n", + "print(len(wcsstep_files), 'assignwcsstep files written')" ] }, { "cell_type": "markdown", - "id": "d4273b91", + "id": "ae590a10-cc87-46a5-adfa-829f67549ec4", "metadata": {}, "source": [ - "#### Manual Check" + "\n", + "#### Flat Field Step\n", + "\n", + "After the assignwcs file is run, we then want to run the `flat_field` step of the pipeline which removes detector artifacts using the flat field reference files. Read more about the step on [jdox](https://jwst-pipeline.readthedocs.io/en/latest/jwst/flatfield/main.html)" ] }, { "cell_type": "code", "execution_count": null, - "id": "f0c1f449", - "metadata": {}, + "id": "b6fd8419-841f-426d-aed2-31e7038c819c", + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "filedir = 'data/NGDEEP/rate/'\n", - "dir_f115w = filedir + 'jw02079004001_04101_00004_nis_flatfieldstep.fits'\n", - "disp_f115w = filedir + 'jw02079004001_05101_00001_nis_flatfieldstep.fits'\n", - "\n", - "dir_f150w = filedir + 'jw02079004002_10101_00004_nis_flatfieldstep.fits'\n", - "disp_f150w = filedir + 'jw02079004002_11101_00001_nis_flatfieldstep.fits'\n", - "\n", - "dir_f200w = filedir + 'jw02079004003_04101_00004_nis_flatfieldstep.fits'\n", - "disp_f200w = filedir + 'jw02079004003_05101_00001_nis_flatfieldstep.fits'" + "# Run flat_field\n", + "for wcsfile in wcsstep_files:\n", + " result = FlatFieldStep.call(wcsfile, output_dir=data_dir_out, save_results=True)" ] }, { "cell_type": "code", "execution_count": null, - "id": "da61c2a4", + "id": "f7a250d7-85c8-4fec-8e7d-929027811ca0", "metadata": {}, "outputs": [], "source": [ - "imviz2 = Imviz()\n", - "with warnings.catch_warnings():\n", - " warnings.simplefilter('ignore')\n", - " imviz2.load_data(dir_f115w, data_label='F115W dir')\n", - " imviz2.load_data(disp_f115w, data_label='F115W wfss')\n", - " imviz2.load_data(dir_f150w, data_label='F150W dir')\n", - " imviz2.load_data(disp_f150w, data_label='F150W wfss')\n", - " imviz2.load_data(dir_f200w, data_label='F200W dir')\n", - " imviz2.load_data(disp_f200w, data_label='F200W wfss')\n", - "imviz2.show()" + "# A quick sanity check to ensure that the files were calibrated.\n", + "# if this is zero, check the log message above for any errors that may have occurred during the calibration\n", + "flatfield_files = glob.glob(os.path.join(data_dir_out, '*flatfieldstep*'))\n", + "print(len(flatfield_files), 'flatfieldstep files written')" ] }, { "cell_type": "markdown", - "id": "037d515a", + "id": "cd2eee20-b5c9-4a77-a9e0-e23eed1baa63", "metadata": {}, "source": [ - "Split direct images and dispersed images in two separate viewers." + "\n", + "#### Compare Rate vs. Flat fielded Data\n", + "Running the cell below shows the same direct image from just the rate file versus when the `flat_field` step of the pipeline is run. Some detector artifacts are noticably gone.\n", + "\n", + "There are remaining optical artifacts to be aware of: https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrument-features-and-caveats" ] }, { "cell_type": "code", "execution_count": null, - "id": "7c8892f4", + "id": "dde1dfc3-c2af-4797-b97f-16ce249212cb", "metadata": {}, "outputs": [], "source": [ - "plotopt = imviz2.plugins['Plot Options']\n", - "plotopt.multiselect = True\n", - "plotopt.select_all(viewers=True, layers=True)\n", - "plotopt.stretch_preset = '99.5%'\n", - "plotopt.multiselect = False" + "test_rate_file = np.sort(list(rate_df[rate_df['FILTER'] == 'CLEAR']['FILENAME']))[0]\n", + "test_flat_file = os.path.join(data_dir_out, os.path.basename(test_rate_file).replace('rate.fits', 'flatfieldstep.fits'))\n", + "\n", + "plot_files = [test_rate_file, test_flat_file]\n", + "plot_titles = ['Rate File', 'Flat Corrected File']\n", + "\n", + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 15))\n", + "fig.suptitle(f\"{os.path.basename(test_rate_file).split('_rate')[0]}\\n{fits.getval(test_rate_file, 'PUPIL')}\", x=0.5, y=0.72)\n", + "\n", + "for filename, title, ax in zip(plot_files, plot_titles, [ax1, ax2]):\n", + " with fits.open(filename) as hdu:\n", + " # fill in the nan values from the bad pixels with zero; otherwise a single, non-dithered image is impossible to really see\n", + " data = hdu[1].data\n", + " data[np.isnan(data)] = 0\n", + " \n", + " ax.imshow(data, vmin=0, vmax=1.7, origin='lower')\n", + " ax.set_title(title)" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "ba0d07ff", + "cell_type": "markdown", + "id": "3cb991cf-af50-4189-8474-19adcb5768d8", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "\"Space" + ] } ], "metadata": { @@ -365,7 +281,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.11.7" } }, "nbformat": 4, From 647d5450e4323d13d4fc9ee41d4d31e2d621d3aa Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 11:44:19 -0500 Subject: [PATCH 18/62] Modifications made while converting the notebooks into other notebooks --- ...D.ipynb => 00_mast_query_data_setup.ipynb} | 0 .../01_check_files_headers_progID.ipynb | 122 +++--------------- .../02_run_pipeline_wcs_flatfield.ipynb | 2 +- 3 files changed, 18 insertions(+), 106 deletions(-) rename notebooks/NIRISS_WFSS_advanced/{00_mast_query_progID.ipynb => 00_mast_query_data_setup.ipynb} (100%) diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/00_mast_query_data_setup.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/00_mast_query_progID.ipynb rename to notebooks/NIRISS_WFSS_advanced/00_mast_query_data_setup.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb index 7671e49cf..d577110b1 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb @@ -10,9 +10,20 @@ "Credit Jo Taylor." ] }, + { + "cell_type": "markdown", + "id": "a8489a5e-7c72-49e4-b666-e4ca21d42031", + "metadata": {}, + "source": [ + "- the dataframe portion of this notebook is now in \"03_imviz_level2\"\n", + "- moving the files was moved to 00_mas_query_progID\n", + "\n", + "Do no use this notebook anymore (depricated)" + ] + }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "nasty-belize", "metadata": {}, "outputs": [], @@ -33,11 +44,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "3db7c79f", - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "# set up an empty dataframe initially to fill in\n", @@ -82,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "a77d81da", "metadata": {}, "outputs": [], @@ -111,109 +120,12 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "6bc1b5c4", "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Moved: data/jw02079004001_04101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004001_04101_00002_nis_rate.fits\n", - "Moved: data/jw02079004003_06101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004003_06101_00002_nis_rate.fits\n", - "Moved: data/jw02079004002_10101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_10101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_06101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_06101_00003_nis_rate.fits\n", - "Moved: data/jw02079004003_02101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004003_02101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_10101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_10101_00002_nis_rate.fits\n", - "Moved: data/jw02079004001_04101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_04101_00001_nis_rate.fits\n", - "Moved: data/jw02079004001_12101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004001_12101_00003_nis_rate.fits\n", - "Moved: data/jw02079004003_06101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004003_06101_00001_nis_rate.fits\n", - "Moved: data/jw02079004003_04101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004003_04101_00002_nis_rate.fits\n", - "Moved: data/jw02079004001_10101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004001_10101_00004_nis_rate.fits\n", - "Moved: data/jw02079004001_06101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004001_06101_00002_nis_rate.fits\n", - "Moved: data/jw02079004002_04101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_04101_00003_nis_rate.fits\n", - "Moved: data/jw02079004001_02101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_02101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_11101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_11101_00002_nis_rate.fits\n", - "Moved: data/jw02079004002_12101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_12101_00001_nis_rate.fits\n", - "Moved: data/jw02079004001_08101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_08101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_11101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_11101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_04101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004002_04101_00004_nis_rate.fits\n", - "Moved: data/jw02079004002_12101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_12101_00002_nis_rate.fits\n", - "Moved: data/jw02079004001_10101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004001_10101_00003_nis_rate.fits\n", - "Moved: data/jw02079004003_04101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004003_04101_00001_nis_rate.fits\n", - "Moved: data/jw02079004001_06101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_06101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_12101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_12101_00003_nis_rate.fits\n", - "Moved: data/jw02079004002_04101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_04101_00001_nis_rate.fits\n", - "Moved: data/jw02079004003_04101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004003_04101_00004_nis_rate.fits\n", - "Moved: data/jw02079004001_10101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004001_10101_00002_nis_rate.fits\n", - "Moved: data/jw02079004001_06101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004001_06101_00003_nis_rate.fits\n", - "Moved: data/jw02079004003_04101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004003_04101_00003_nis_rate.fits\n", - "Moved: data/jw02079004001_10101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_10101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_11101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_11101_00003_nis_rate.fits\n", - "Moved: data/jw02079004002_04101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_04101_00002_nis_rate.fits\n", - "Moved: data/jw02079004002_06101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_06101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_10101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004002_10101_00003_nis_rate.fits\n", - "Moved: data/jw02079004001_04101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004001_04101_00004_nis_rate.fits\n", - "Moved: data/jw02079004001_12101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004001_12101_00002_nis_rate.fits\n", - "Moved: data/jw02079004003_06101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004003_06101_00003_nis_rate.fits\n", - "Moved: data/jw02079004002_02101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_02101_00001_nis_rate.fits\n", - "Moved: data/jw02079004002_08101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004002_08101_00001_nis_rate.fits\n", - "Moved: data/jw02079004001_12101_00001_nis_rate.fits to data/NGDEEP/rate/jw02079004001_12101_00001_nis_rate.fits\n", - "Moved: data/jw02079004001_04101_00003_nis_rate.fits to data/NGDEEP/rate/jw02079004001_04101_00003_nis_rate.fits\n", - "Moved: data/jw02079004002_10101_00004_nis_rate.fits to data/NGDEEP/rate/jw02079004002_10101_00004_nis_rate.fits\n", - "Moved: data/jw02079004002_06101_00002_nis_rate.fits to data/NGDEEP/rate/jw02079004002_06101_00002_nis_rate.fits\n", - "Unrecognized index: spec2 or asn\n", - "Moved: data/jw02079-o004_20231008t143836_image3_00005_asn.json to data/NGDEEP/asn_level3/jw02079-o004_20231008t143836_image3_00005_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00623_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00623_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00444_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00444_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00740_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00740_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00550_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00550_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00551_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00551_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00031_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00031_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00698_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00698_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00699_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00699_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00329_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00329_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00328_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00328_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00106_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00106_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00107_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00107_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00148_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00148_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00296_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00296_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image3_00009_asn.json to data/NGDEEP/asn_level3/jw02079-o004_20231008t143836_image3_00009_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00476_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00476_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00477_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00477_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00401_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00401_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00400_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00400_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image3_00002_asn.json to data/NGDEEP/asn_level3/jw02079-o004_20231008t143836_image3_00002_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00624_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00624_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00625_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00625_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00255_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00255_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00033_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00033_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00032_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00032_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00254_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00254_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00105_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00105_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00104_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00104_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00179_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00179_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00180_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00180_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00181_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00181_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00549_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00549_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00548_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00548_asn.json\n", - "Unrecognized index: spec2 or asn\n", - "Unrecognized index: spec2 or asn\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00592_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00592_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00475_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00475_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00402_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00402_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00403_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00403_asn.json\n", - "Unrecognized index: spec3 or asn\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00252_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00252_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00253_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00253_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00696_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00696_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00697_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00697_asn.json\n", - "Moved: data/jw02079-o004_20231008t143836_image2_00327_asn.json to data/NGDEEP/asn_level2/jw02079-o004_20231008t143836_image2_00327_asn.json\n" - ] - } - ], + "outputs": [], "source": [ "# first, make all of the new directories\n", "topdir = 'data/NGDEEP'\n", diff --git a/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb b/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb index 0251f1de1..d839d8cd7 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb @@ -183,7 +183,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.11.7" } }, "nbformat": 4, From 4b871fb3d831e7124c4b6bd5a11665dde711c096 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 11:45:37 -0500 Subject: [PATCH 19/62] removing notebooks that have been moved into other notebooks --- .../01_check_files_headers_progID.ipynb | 194 ------------------ .../02_run_pipeline_wcs_flatfield.ipynb | 191 ----------------- 2 files changed, 385 deletions(-) delete mode 100644 notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb delete mode 100644 notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb b/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb deleted file mode 100644 index d577110b1..000000000 --- a/notebooks/NIRISS_WFSS_advanced/01_check_files_headers_progID.ipynb +++ /dev/null @@ -1,194 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "continental-machinery", - "metadata": {}, - "source": [ - "# Create list of files from MAST download\n", - "\n", - "Credit Jo Taylor." - ] - }, - { - "cell_type": "markdown", - "id": "a8489a5e-7c72-49e4-b666-e4ca21d42031", - "metadata": {}, - "source": [ - "- the dataframe portion of this notebook is now in \"03_imviz_level2\"\n", - "- moving the files was moved to 00_mas_query_progID\n", - "\n", - "Do no use this notebook anymore (depricated)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "nasty-belize", - "metadata": {}, - "outputs": [], - "source": [ - "from astropy.io import fits\n", - "import pandas as pd\n", - "import glob\n", - "import os" - ] - }, - { - "cell_type": "markdown", - "id": "95d38a86", - "metadata": {}, - "source": [ - "## Get list of files with header keywords" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3db7c79f", - "metadata": {}, - "outputs": [], - "source": [ - "# set up an empty dataframe initially to fill in\n", - "df = pd.DataFrame(columns=[\"FILENAME\",\n", - " \"TARG_RA\", \n", - " \"TARG_DEC\", \n", - " \"FILTER\", # Grism; GR150R/GR150C\n", - " \"PUPIL\", # Filter used; F090W, F115W, F140M, F150W F158M, F200W\n", - " \"PATT_NUM\", # Position number within dither pattern for WFSS\n", - " \"NUMDTHPT\", # Total number of points in entire dither pattern\n", - " \"XOFFSET\", # X offset from pattern starting position for NIRISS (arcsec)\n", - " \"YOFFSET\"]) # Y offset from pattern starting position for NIRISS (arcsec)\n", - "\n", - "for ratefile in glob.glob(\"data/*rate.fits\"):\n", - " image = fits.open(ratefile)\n", - " \n", - " # create a new dataframe for each file\n", - " df2 = pd.DataFrame({\"FILENAME\" : [image[0].header[\"FILENAME\"]],\n", - " \"TARG_RA\" : [image[0].header[\"TARG_RA\"]],\n", - " \"TARG_DEC\" : [image[0].header[\"TARG_DEC\"]],\n", - " \"FILTER\" : [image[0].header[\"FILTER\"]],\n", - " \"PUPIL\" : [image[0].header[\"PUPIL\"]],\n", - " \"PATT_NUM\" : [image[0].header[\"PATT_NUM\"]],\n", - " \"NUMDTHPT\" : [image[0].header[\"NUMDTHPT\"]],\n", - " \"XOFFSET\" : [image[0].header[\"XOFFSET\"]],\n", - " \"YOFFSET\" : [image[0].header[\"YOFFSET\"]]\n", - " })\n", - " \n", - " # merge the two dataframes together to create a dataframe with all \n", - " df = pd.concat([df, df2], ignore_index = True, axis = 0)\n", - " \n", - "dfsort = df.sort_values('FILENAME', ignore_index = True)" - ] - }, - { - "cell_type": "markdown", - "id": "7002e5c9", - "metadata": {}, - "source": [ - "## Save to csv file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a77d81da", - "metadata": {}, - "outputs": [], - "source": [ - "dfsort.to_csv(\"./list_ngdeep_rate.csv\",sep=',')" - ] - }, - { - "cell_type": "markdown", - "id": "719e5a24", - "metadata": {}, - "source": [ - "## Move files to preferred location" - ] - }, - { - "cell_type": "markdown", - "id": "12a195ca", - "metadata": {}, - "source": [ - "At this point, I move the rate files and the association files to a preferred place on my machine:\n", - "- all rate files under NGDEEP/rate\n", - "- level 2 association files under NGDEEP/asn_level2\n", - "- level 3 association files under NGDEEP/asn_level3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6bc1b5c4", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# first, make all of the new directories\n", - "topdir = 'data/NGDEEP'\n", - "if not os.path.exists(topdir):\n", - " os.mkdir(topdir)\n", - " print('Created:', topdir)\n", - "\n", - "for subdir in ['rate', 'asn_level2', 'asn_level3']:\n", - " new_subdir = os.path.join(topdir, subdir)\n", - " if not os.path.exists(new_subdir):\n", - " os.mkdir(new_subdir)\n", - " print('Created:', new_subdir)\n", - "\n", - "file_dict = {'rate' : 'rate',\n", - " 'image3' : 'asn_level3',\n", - " 'image2' : 'asn_level2'}\n", - "\n", - "# now move all of the files to the appropriate locations\n", - "for filename in glob.glob('data/*.fits') + glob.glob('data/*.json'):\n", - " try:\n", - " index = filename.split('_')[2] # json files; looking for image2/image3\n", - " subdir = file_dict[index]\n", - " except KeyError:\n", - " try:\n", - " index2 = filename.split('_')[-1].split('.')[0] # rate files\n", - " subdir = file_dict[index2]\n", - " except KeyError:\n", - " print(f'Unrecognized index: {index} or {index2}')\n", - " continue\n", - " \n", - " new_file = os.path.join(topdir, subdir, os.path.basename(filename))\n", - " os.rename(filename, new_file)\n", - " print(f'Moved: {filename} to {new_file}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "935d05d9", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb b/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb deleted file mode 100644 index d839d8cd7..000000000 --- a/notebooks/NIRISS_WFSS_advanced/02_run_pipeline_wcs_flatfield.ipynb +++ /dev/null @@ -1,191 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1cea74b7", - "metadata": {}, - "source": [ - "# Run assign_wcs and flatfielding" - ] - }, - { - "cell_type": "markdown", - "id": "5e0e0498", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "id": "da56a9fb", - "metadata": {}, - "source": [ - "\n", - "## Imports" - ] - }, - { - "cell_type": "raw", - "id": "58db78a6", - "metadata": {}, - "source": [ - "# Update the CRDS path to your local directory\n", - "%env CRDS_PATH= (/grp/crds/jwst/)\n", - "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ee53c65", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import glob\n", - "#jwst pipeline related modules\n", - "import jwst\n", - "from jwst import datamodels\n", - "# Individual steps that make up calwebb_image2\n", - "from jwst.background import BackgroundStep\n", - "from jwst.assign_wcs import AssignWcsStep\n", - "from jwst.flatfield import FlatFieldStep\n", - "from jwst.photom import PhotomStep\n", - "from jwst.resample import ResampleStep" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df3bb687", - "metadata": {}, - "outputs": [], - "source": [ - "print('jwst:', jwst.__version__)" - ] - }, - { - "cell_type": "markdown", - "id": "aa57e27d", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4084d5a5", - "metadata": {}, - "outputs": [], - "source": [ - "# Directory definitions. The rate files should be located here based on notebooks 00 & 01\n", - "data_dir_in = \"data/NGDEEP/rate/*\"\n", - "data_dir_out = \"data/NGDEEP/wcs_flat/\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "97183293", - "metadata": {}, - "outputs": [], - "source": [ - "# List the rate files\n", - "ratefiles = glob.glob(data_dir_in+'*')" - ] - }, - { - "cell_type": "raw", - "id": "66bc6d0a", - "metadata": {}, - "source": [ - "# Check the name, pupil, and filter of all files (this step is not necessary, just a sanity check)\n", - "for file in ratefiles:\n", - " model = datamodels.open(file)\n", - " print(file)\n", - " print(model.meta.instrument.name)\n", - " print(model.meta.instrument.pupil)\n", - " print(model.meta.instrument.filter)\n", - " print()\n", - " model.close()" - ] - }, - { - "cell_type": "markdown", - "id": "2e145cf6", - "metadata": {}, - "source": [ - "**Developer note:** it would be better to update these to use the `call()` method instead of the `run()` method." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b143bddf", - "metadata": {}, - "outputs": [], - "source": [ - "# Run assign_wcs\n", - "awcsfiles = []\n", - "for file in ratefiles:\n", - " model = datamodels.open(file)\n", - " name = file.split('/')[-1]\n", - " newname = name.replace('rate.fits', 'assignwcsstep.fits')\n", - " assign_wcs_step = AssignWcsStep()\n", - " assign_wcs_step.output_dir = data_dir_out\n", - " assign_wcs_step.save_results = True\n", - " #run the step\n", - " assign_wcs_step.run(file)\n", - " awcsfiles.append(data_dir_out+newname)\n", - " print(newname)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e6e6f72a", - "metadata": {}, - "outputs": [], - "source": [ - "# Run flatfielding\n", - "fffiles = []\n", - "for file in awcsfiles:\n", - " newname = file.replace('assignwcsstep.fits', 'flatfieldstep.fits')\n", - " flatfield_step = FlatFieldStep()\n", - " flatfield_step.output_dir = data_dir_out\n", - " flatfield_step.save_results = True \n", - " #run the step\n", - " flatfield_step.run(file)\n", - " fffiles.append(newname)\n", - " print(newname)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f46c65fa", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From aeab6ee9b57416203f97c6f0c73106d71aa649fc Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 11:46:52 -0500 Subject: [PATCH 20/62] removing notebook 05 that has been moved into notebook 04 (soon to be 01) --- .../05_catalog_for_spectral_extraction.ipynb | 288 ------------------ 1 file changed, 288 deletions(-) delete mode 100644 notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb b/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb deleted file mode 100644 index c5c1336e3..000000000 --- a/notebooks/NIRISS_WFSS_advanced/05_catalog_for_spectral_extraction.ipynb +++ /dev/null @@ -1,288 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5e3e6ab1", - "metadata": {}, - "source": [ - "# Create catalog for spectral extraction\n", - "\n", - "Create catalogs for each filter with matching IDs for easy analysis after the extraction of the 2D spectra. No filtering is applied here. Make copy of original catalogs because they will be overwritten. If I overwrite the catalogs, I do not have to change all the association files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d9602eab", - "metadata": {}, - "outputs": [], - "source": [ - "import glob\n", - "import shutil\n", - "from astropy.table import Table\n", - "from astropy.coordinates import SkyCoord\n", - "from astropy import units as u\n", - "from jdaviz import Imviz" - ] - }, - { - "cell_type": "markdown", - "id": "aa194c49", - "metadata": {}, - "source": [ - "## Read catalogs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cfadc316", - "metadata": {}, - "outputs": [], - "source": [ - "data_dir = '/Users/cpacifici/DATA/NGDEEP/image3/'\n", - "file_f115w = data_dir + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv'\n", - "file_f150w = data_dir + 'jw02079-o004_t001_niriss_clear-f150w_cat.ecsv'\n", - "file_f200w = data_dir + 'jw02079-o004_t001_niriss_clear-f200w_cat.ecsv'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8d711d91", - "metadata": {}, - "outputs": [], - "source": [ - "cat_f115w = Table.read(file_f115w)\n", - "cat_f150w = Table.read(file_f150w)\n", - "cat_f200w = Table.read(file_f200w)\n", - "print(len(cat_f150w))\n", - "cat_f150w['sky_centroid']" - ] - }, - { - "cell_type": "markdown", - "id": "c9d738f3", - "metadata": {}, - "source": [ - "Make copy of the catalogs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d414e56b", - "metadata": {}, - "outputs": [], - "source": [ - "shutil.copy(file_f115w, data_dir+'jw02079-o004_t001_niriss_clear-f115w_cat_original.ecsv')\n", - "shutil.copy(file_f150w, data_dir+'jw02079-o004_t001_niriss_clear-f150w_cat_original.ecsv')\n", - "shutil.copy(file_f200w, data_dir+'jw02079-o004_t001_niriss_clear-f200w_cat_original.ecsv')" - ] - }, - { - "cell_type": "markdown", - "id": "81a83a9b", - "metadata": {}, - "source": [ - "### Plot in Imviz" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f50c9286", - "metadata": {}, - "outputs": [], - "source": [ - "imagefile = data_dir + 'jw02079-o004_t001_niriss_clear-f150w_i2d.fits'\n", - "imviz = Imviz()\n", - "imviz.load_data(imagefile, data_label='F150W')\n", - "imviz.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47b4217a", - "metadata": {}, - "outputs": [], - "source": [ - "# Plot markers for all objects\n", - "viewer = imviz.default_viewer\n", - "viewer.reset_markers()\n", - "t_f115w = Table({'coord': cat_f115w['sky_centroid']})\n", - "t_f150w = Table({'coord': cat_f150w['sky_centroid']})\n", - "t_f200w = Table({'coord': cat_f200w['sky_centroid']})\n", - "viewer.marker = {'color': 'green', 'alpha': 0.8, 'markersize': 30, 'fill': False}\n", - "viewer.add_markers(t_f115w, use_skycoord=True, marker_name='full_cat_f115w')\n", - "viewer.marker = {'color': 'orange', 'alpha': 0.8, 'markersize': 5, 'fill': True}\n", - "viewer.add_markers(t_f150w, use_skycoord=True, marker_name='full_cat_f150w')\n", - "viewer.marker = {'color': 'red', 'alpha': 0.8, 'markersize': 5, 'fill': True}\n", - "viewer.add_markers(t_f200w, use_skycoord=True, marker_name='full_cat_f200w')" - ] - }, - { - "cell_type": "markdown", - "id": "3da1408d", - "metadata": {}, - "source": [ - "## Rewrite catalogs with matching IDs" - ] - }, - { - "cell_type": "markdown", - "id": "587b4376", - "metadata": {}, - "source": [ - "### Match catalogs and find object in all three\n", - "\n", - "1. Match F115W to F150W -> f115w_base_matches\n", - "2. Match f115w_base_matches to F200W -> f115w_base_matches, f200w_try_matches\n", - "3. Rematch the matched F115W to F150W -> f115w_matches, f150w_try_matches" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4048f182", - "metadata": {}, - "outputs": [], - "source": [ - "f115w_base = cat_f115w['sky_centroid']\n", - "f150w_try = cat_f150w['sky_centroid']\n", - "\n", - "max_sep = 1.0 * u.arcsec\n", - "idx, d2d, d3d = f115w_base.match_to_catalog_3d(f150w_try)\n", - "sep_constraint = d2d < max_sep\n", - "f115w_base_matches = cat_f115w[sep_constraint]\n", - "f150w_try_matches = cat_f150w[idx[sep_constraint]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b8956cca", - "metadata": {}, - "outputs": [], - "source": [ - "f115w_base = f115w_base_matches['sky_centroid']\n", - "f200w_try = cat_f200w['sky_centroid']\n", - "\n", - "max_sep = 1.0 * u.arcsec\n", - "idx, d2d, d3d = f115w_base.match_to_catalog_3d(f200w_try)\n", - "sep_constraint = d2d < max_sep\n", - "f115w_base_matches = f115w_base_matches[sep_constraint]\n", - "f200w_try_matches = cat_f200w[idx[sep_constraint]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "65915dfd", - "metadata": {}, - "outputs": [], - "source": [ - "f115w_base = f115w_base_matches['sky_centroid']\n", - "f150w_try = cat_f150w['sky_centroid']\n", - "\n", - "max_sep = 1.0 * u.arcsec\n", - "idx, d2d, d3d = f115w_base.match_to_catalog_3d(f150w_try)\n", - "sep_constraint = d2d < max_sep\n", - "f115w_matches = f115w_base_matches[sep_constraint]\n", - "f150w_try_matches = cat_f150w[idx[sep_constraint]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8e1da323", - "metadata": {}, - "outputs": [], - "source": [ - "f115w_matches" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b2034693", - "metadata": {}, - "outputs": [], - "source": [ - "# Check that they all have the same length\n", - "print(\"Length F115W match: \", len(f115w_matches))\n", - "print(\"Length F150W match: \", len(f150w_try_matches))\n", - "print(\"Length F200W match: \", len(f200w_try_matches))" - ] - }, - { - "cell_type": "markdown", - "id": "e36d9525", - "metadata": {}, - "source": [ - "### Rewrite catalogs" - ] - }, - { - "cell_type": "markdown", - "id": "516f62fa", - "metadata": {}, - "source": [ - "Overwrite. This is the critical step. Use the same `label` for the same targets in the three catalogs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1ae2d0ab", - "metadata": {}, - "outputs": [], - "source": [ - "f150w_try_matches['label'] = f115w_matches['label']\n", - "f200w_try_matches['label'] = f115w_matches['label']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30362634", - "metadata": {}, - "outputs": [], - "source": [ - "f115w_matches.write(file_f115w, format='ascii.ecsv', overwrite=True)\n", - "f150w_try_matches.write(file_f150w, format='ascii.ecsv', overwrite=True)\n", - "f200w_try_matches.write(file_f200w, format='ascii.ecsv', overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5eba8956", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 3a896426b96b3a8b8089e515b457687346df7de3 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 11:48:41 -0500 Subject: [PATCH 21/62] moving around some text & imports --- .../04_run_pipeline_imaging_level2and3.ipynb | 54 +++++++++---------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb index c42a18067..3e1063a6f 100644 --- a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb @@ -5,13 +5,13 @@ "id": "7a935d7c", "metadata": {}, "source": [ - "# Run Image pipeline and create catalog of sources\n", + "# Run Image Pipeline and Create Source Catalog\n", "\n", - "The first calibration that should be done as part of a WFSS run is to run the direct images through the Image2 and Image3 steps of the JWST pipeline. This includes creating a source catalog, which most likely will need to be adjusted from the pipeline default values. **Not having a good source catalog will result in non optimal extraction of sources in the dispersed, WFSS, images.**\n", + "In this run, we are starting from the rate files which have already been calibrated with the detector1 pipeline. This will save time as we do not need to edit any of the steps being performed as part of detector1. Therefore, the first calibration that should be done as part of a WFSS run is to run the rate files direct images through the Image2 and Image3 steps of the JWST pipeline. This includes creating a source catalog, which most likely will need to be adjusted from the pipeline default values. **Not having a good source catalog will result in non optimal extraction of sources in the dispersed, WFSS, images.**\n", "\n", "**Use case**: The default parameters for the pipeline do not extract the expected sources, so a custom parameters need to be set to obtain new combined image and source catalog.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, shutil, warnings
\n", + "**Tools**: astropy, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, warnings
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -50,29 +50,22 @@ "import os\n", "import glob\n", "import json\n", - "import shutil\n", "import warnings\n", "import numpy as np\n", "import pandas as pd\n", + "\n", "import astropy.units as u\n", "from astropy.io import fits\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Table\n", + "\n", + "from jdaviz import Imviz\n", "from matplotlib import pyplot as plt\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bd8dc296-44cc-4118-a036-5a9ac68c456f", - "metadata": {}, - "outputs": [], - "source": [ + "%matplotlib inline\n", + "\n", "import jwst\n", "from jwst.pipeline import Image2Pipeline\n", - "from jwst.pipeline import Image3Pipeline\n", - "from jdaviz import Imviz" + "from jwst.pipeline import Image3Pipeline" ] }, { @@ -85,6 +78,14 @@ "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" ] }, + { + "cell_type": "markdown", + "id": "fb035a4d-e63c-4125-b083-462e3353e49f", + "metadata": {}, + "source": [ + "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available, check here" + ] + }, { "cell_type": "code", "execution_count": null, @@ -107,17 +108,6 @@ "custom_run_image3 = 'custom_image3_calibrated'# where the results of the custom image3 run will be saved (inside of data_dir)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d576fd0", - "metadata": {}, - "outputs": [], - "source": [ - "listrate_file = './list_ngdeep_rate.csv'\n", - "rate_df = pd.read_csv(listrate_file)" - ] - }, { "cell_type": "markdown", "id": "d991e323-8e98-4487-a8ab-1b696f91899b", @@ -897,10 +887,12 @@ "metadata": {}, "outputs": [], "source": [ + "# first, look at the current, custom source catalog for one of the filters\n", "cat_name = np.sort(glob.glob(os.path.join(custom_run_image3, '*source-match_cat.ecsv')))[3]\n", - "print(cat_name)\n", "cat = Table.read(cat_name)\n", - "cat[['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']]" + "\n", + "print(cat_name)\n", + "cat" ] }, { @@ -911,7 +903,9 @@ "There may be multiple ways to look for a source, so shown below are three options:\n", "1. With a known RA/Dec of an object\n", "2. A known x/y location of an object\n", - "3. With a source ID of an object. Note that we are using the rebased source catalogs here for the IDs." + "3. With a source ID of an object. Note that we are using the rebased source catalogs here for the IDs.\n", + "\n", + "This same concept can be extended to filter by any of the columns contained in the source catalog." ] }, { From 19b5cc37d5c1f0d3cecc6141470446f2623498da Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 11:50:09 -0500 Subject: [PATCH 22/62] Renaming/reordering notebooks --- ...eline_imaging_level2and3.ipynb => 01_wfss_image2_image3.ipynb} | 0 .../{03_imviz_level2.ipynb => extra_individual_steps.ipynb} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename notebooks/NIRISS_WFSS_advanced/{04_run_pipeline_imaging_level2and3.ipynb => 01_wfss_image2_image3.ipynb} (100%) rename notebooks/NIRISS_WFSS_advanced/{03_imviz_level2.ipynb => extra_individual_steps.ipynb} (100%) diff --git a/notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_wfss_image2_image3.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/04_run_pipeline_imaging_level2and3.ipynb rename to notebooks/NIRISS_WFSS_advanced/01_wfss_image2_image3.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_individual_steps.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/03_imviz_level2.ipynb rename to notebooks/NIRISS_WFSS_advanced/extra_individual_steps.ipynb From 1b29db797346d234496d40a537ee5524099382d5 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 11:51:53 -0500 Subject: [PATCH 23/62] adding 'niriss' into the notebook names --- ...ery_data_setup.ipynb => 00_niriss_mast_query_data_setup.ipynb} | 0 ...fss_image2_image3.ipynb => 01_niriss_wfss_image2_image3.ipynb} | 0 ...individual_steps.ipynb => extra_niriss_individual_steps.ipynb} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename notebooks/NIRISS_WFSS_advanced/{00_mast_query_data_setup.ipynb => 00_niriss_mast_query_data_setup.ipynb} (100%) rename notebooks/NIRISS_WFSS_advanced/{01_wfss_image2_image3.ipynb => 01_niriss_wfss_image2_image3.ipynb} (100%) rename notebooks/NIRISS_WFSS_advanced/{extra_individual_steps.ipynb => extra_niriss_individual_steps.ipynb} (100%) diff --git a/notebooks/NIRISS_WFSS_advanced/00_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/00_mast_query_data_setup.ipynb rename to notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/01_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/01_wfss_image2_image3.ipynb rename to notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/extra_individual_steps.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/extra_individual_steps.ipynb rename to notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb From 7135641ff07ad8c8ceab85b36478990ffb06d4d4 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 18:50:16 -0500 Subject: [PATCH 24/62] Major modifications to spec2 run & visualization --- .../06_run_pipeline_spec2.ipynb | 956 ++++++++++++++++-- 1 file changed, 853 insertions(+), 103 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb index d61832bfb..d96c1f686 100644 --- a/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb @@ -5,13 +5,28 @@ "id": "6d674dc1", "metadata": {}, "source": [ - "# Run spec2 pipeline on rate files\n", + "# Run spec2 pipeline\n", "\n", - "Need to copy all the needed files over to the working directory.\n", + "The dispersed images for WFSS will be run through the spec2 pipeline after the image2 and image3 pipelines have been run on the corresponding direct images. As mentioned in the previous notebook, it is extremely helpful for this step if you have a source catalog that only includes the sources that you would like to look at. Any additional sources will take extra time to calibrate.\n", "\n", - "**Author notes**:\n", - "- Anything I need to know on the spec2 steps?\n", - "- I have to try turning on the correction for contamination" + "**Use case**: After creating a custom source catalog, spec2 should be run on the dispersed WFSS images.
\n", + "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", + "**Tools**: astropy, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, shutil
\n", + "**Cross-instrument**: NIRISS
\n", + "\n", + "**Content**\n", + "- [Imports & Data Setup](#imports)\n", + "- [Custom Spec2 Pipeline Run](#custom_spec2)\n", + " - [Spec2 Association Files](#spec2_asn)\n", + " - [Run Spec2](#spec2_run)\n", + " - [Examine the Outputs of Spec2](#spec2_examine)\n", + "- [Explore Data Further](#explore)\n", + " - [Find a Source](#source)\n", + " - [Limit the Number of Extracted Sources](#limit_source)\n", + " - [Final Visualization](#final_visualize)\n", + "\n", + "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", + "**Last modified**: February 2024" ] }, { @@ -19,266 +34,1001 @@ "id": "b28e541b", "metadata": {}, "source": [ - "## Imports" + "\n", + "## Imports & Data Setup" ] }, { "cell_type": "code", "execution_count": null, - "id": "453e9031", + "id": "9443f2ff", "metadata": {}, "outputs": [], "source": [ - "# Packages that allow us to get information about objects:\n", + "import glob\n", + "import json\n", "import os\n", - "# Update to a path in your system (see details below at \"Reference files\")\n", - "os.environ[\"CRDS_PATH\"] = \"/Users/cpacifici/crds_cache_july2023/\"\n", - "os.environ[\"CRDS_SERVER_URL\"] = \"https://jwst-crds.stsci.edu\"\n" + "import shutil\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from astropy.table import Table\n", + "from astropy.io import fits\n", + "\n", + "from jdaviz import Mosviz\n", + "from matplotlib import pyplot as plt\n", + "from matplotlib import patches\n", + "%matplotlib inline\n", + "\n", + "from jwst.pipeline import Spec2Pipeline\n", + "from jwst import datamodels" + ] + }, + { + "cell_type": "raw", + "id": "41c4c7c2-7914-4f5f-ad5f-4b876bc63555", + "metadata": {}, + "source": [ + "# Update the CRDS path to your local directory\n", + "%env CRDS_PATH= (/grp/crds/jwst/)\n", + "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" + ] + }, + { + "cell_type": "markdown", + "id": "64b6e8d7-64a0-4a3b-9776-29b181452ff9", + "metadata": {}, + "source": [ + "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available, check here" ] }, { "cell_type": "code", "execution_count": null, - "id": "9443f2ff", + "id": "87cdec65", "metadata": {}, "outputs": [], "source": [ - "# The calwebb_spec and spec3 pipelines\n", - "from jwst.pipeline import Spec2Pipeline\n", + "import jwst\n", + "print('jwst',jwst.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "436ed7e4-f642-41d0-83dd-565c24f78d5c", + "metadata": {}, + "source": [ + "#### Data Setup\n", "\n", - "# data models\n", - "from jwst import datamodels\n", + "For spec2, we need the rate files that we downloaded as well as the segmentation maps and source catalogs from image3. Because of that, we will create a new directory, change into it, and copy the relevant data over. \n", "\n", - "# utility\n", - "from astropy.table import Table\n", - "import numpy as np\n", - "import json\n", - "import glob\n", - "import shutil\n", + "*For a regular workflow, it is likely easier to leave all files in a single directory compared to the multiple directories we make here.*" + ] + }, + { + "cell_type": "markdown", + "id": "d218c8e0-7a25-4688-b904-c4225a66a8ca", + "metadata": {}, + "source": [ + "Open up our data information file which will make parsing the rate files to copy over easier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "168c2146", + "metadata": {}, + "outputs": [], + "source": [ + "data_dir = 'data'\n", + "custom_run_spec2 = 'custom_spec2'\n", + "custom_run_image3 = os.path.join(data_dir, 'custom_image3_calibrated') # results of custom image3 calibration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3f8222a-92f1-476a-9dee-85604e8415d0", + "metadata": {}, + "outputs": [], + "source": [ + "# From the csv file we created earlier, find a list of all of the grism observations we will want to calibrate with spec2\n", + "listrate_file = 'list_ngdeep_rate.csv'\n", + "rate_df = pd.read_csv(listrate_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bfd083d-55b5-4610-b11e-276388394e27", + "metadata": {}, + "outputs": [], + "source": [ + "# copy all of the grism rate files\n", + "gr150r_files = list(rate_df[rate_df['FILTER'] == 'GR150R']['FILENAME'])\n", + "gr150c_files = list(rate_df[rate_df['FILTER'] == 'GR150C']['FILENAME'])\n", "\n", - "# jdaviz\n", - "from jdaviz import Mosviz" + "for grism_rate in gr150r_files + gr150c_files:\n", + " if os.path.exists(grism_rate):\n", + " shutil.copy(grism_rate, os.path.join(data_dir, custom_run_spec2, os.path.basename(grism_rate)))\n", + " else:\n", + " print(f'{grism_rate} does not exist. Not able to copy file.')" ] }, { "cell_type": "code", "execution_count": null, - "id": "87cdec65", + "id": "e03967b5-e6d1-4cd6-a0ca-d429b5fa18c6", "metadata": {}, "outputs": [], "source": [ - "import jwst\n", - "print('jwst',jwst.__version__)" + "# copy all of the spec2 asn files\n", + "for asn in glob.glob(os.path.join(data_dir, '*spec2*asn*.json')):\n", + " if os.path.exists(asn):\n", + " shutil.copy(asn, os.path.join(data_dir, custom_run_spec2, os.path.basename(asn)))\n", + " else:\n", + " print(f'{asn} does not exist. Not able to copy file.')" ] }, { "cell_type": "code", "execution_count": null, - "id": "168c2146", + "id": "d5beb0fe-ff62-453c-95b5-046d6f2aec10", + "metadata": {}, + "outputs": [], + "source": [ + "# copy all of the necessary image3 output files\n", + "cats = glob.glob(os.path.join(custom_run_image3, '*source*_cat.ecsv')) # copy both the source-match and source155 catalogs\n", + "segm = glob.glob(os.path.join(custom_run_image3, '*_segm.fits'))\n", + "\n", + "for image3_file in cats + segm:\n", + " if os.path.exists(image3_file):\n", + " shutil.copy(image3_file, os.path.join(data_dir, custom_run_spec2, os.path.basename(image3_file)))\n", + " else:\n", + " print(f'{image3_file} does not exist. Not able to copy file.')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c02d9212-7f9a-4f8d-a769-6765bc5bf710", "metadata": {}, "outputs": [], "source": [ - "# Update appropriately\n", - "data_dir_in = \"/Users/cpacifici/DATA/NGDEEP/rate/\"\n", - "data_dir_images = \"/Users/cpacifici/DATA/NGDEEP/image3/\"\n", - "data_dir_asn_level2 = \"/Users/cpacifici/DATA/NGDEEP/asn_level2/\"\n", - "data_dir_out_spec2 = \"/Users/cpacifici/DATA/NGDEEP/spec2/\"" + "cwd = os.getcwd() # get the current working directory \n", + "if cwd != data_dir: # if you are not already in the location of the data, change into it\n", + " try:\n", + " os.chdir(data_dir)\n", + " except FileNotFoundError:\n", + " print(f\"Not able to change into: {data_dir}.\\nRemaining in: {cwd}\")\n", + " pass\n", + "\n", + "if not os.path.exists(custom_run_spec2):\n", + " os.mkdir(custom_run_spec2)\n", + "\n", + "os.chdir(custom_run_spec2)\n", + "\n", + "new_cwd = os.getcwd()\n", + "print('Now in:', new_cwd)" + ] + }, + { + "cell_type": "markdown", + "id": "e0a176fd-094d-44e5-b860-5b9f730e0898", + "metadata": {}, + "source": [ + "\n", + "## Custom Spec2 Pipeline Run\n", + "\n", + "Because this is a custom spec2 pipeline run, the first step is to make sure the correct source catalog is being used. This will define the spectral trace cutouts, and therefore inform the pipeline on where to extract the spectrum." + ] + }, + { + "cell_type": "markdown", + "id": "aee8bd1b-1012-47ea-8ce4-fece4c2866a0", + "metadata": {}, + "source": [ + "\n", + "### Spec2 Association Files\n", + "\n", + "As with the imaging part of the pipeline, there are association files for spec2. These are a bit more complex in that they need to have the science (WFSS) data, direct image, source catalog, and segmentation map included as members. " ] }, { "cell_type": "markdown", - "id": "1aa37d93", + "id": "7228f9ea-5e57-41d3-b236-69d7de008423", "metadata": {}, "source": [ - "## Read list of rate files for imaging" + "#### Looking in a Spec2 Association File" ] }, { "cell_type": "code", "execution_count": null, - "id": "8b5572ab", + "id": "bf6515ea-2b60-4cb1-92ab-3b8dd72a8c01", "metadata": {}, "outputs": [], "source": [ - "listrate_file = './list_ngdeep_rate.csv'\n", - "listrate = Table.read(listrate_file)\n", - "listrate[np.where((listrate['FILTER']=='GR150R') | (listrate['FILTER']=='GR150C'))]" + "spec2_asns = np.sort(glob.glob('*spec2*asn*.json'))\n", + "print(len(spec2_asns), 'Spec2 ASN files')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87e4b9de-b857-4c2b-a58d-a90ef2aadf05", + "metadata": {}, + "outputs": [], + "source": [ + "## look at one of the association files\n", + "asn_data = json.load(open(spec2_asns[0]))\n", + "for key, data in asn_data.items():\n", + " print(f\"{key} : {data}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41c5c5a0-0641-4b7d-bf0c-f1ce688c48e1", + "metadata": {}, + "outputs": [], + "source": [ + "## in particular, take a closer look at the product filenames with the association file:\n", + "for product in asn_data['products']:\n", + " for key, value in product.items():\n", + " if key == 'members':\n", + " print(f\"{key}:\")\n", + " for member in value:\n", + " print(f\" {member['expname']} : {member['exptype']}\")\n", + " else:\n", + " print(f\"{key}: {value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "5caee78b-b2e4-432d-9838-836495f390ac", + "metadata": {}, + "source": [ + "#### Modify the Association File to use Custom Source Catalog\n", + "\n", + "What is currently in the association files uses the pipeline source catalog. In the previous notebook, we created a custom source catalog that we want to use with the extension `source-match_cat.ecsv`, so we need to point to that catalog instead in the association files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4074a86-fd34-42e7-b09f-bf6e66c4b866", + "metadata": {}, + "outputs": [], + "source": [ + "new_source_ext = 'source-match_cat.ecsv'\n", + "\n", + "# loop through all of the spec2 association files in the current directory\n", + "for asn in spec2_asns:\n", + " asn_data = json.load(open(asn))\n", + " for product in asn_data['products']: # for every product, check the members \n", + " for member in product['members']: # there are multiple members per product\n", + " if member['exptype'] == 'sourcecat':\n", + " cat_in_asn = member['expname']\n", + " # check that we haven't already updated the source catalog name\n", + " if new_source_ext not in cat_in_asn:\n", + " new_cat = cat_in_asn.replace('cat.ecsv', new_source_ext)\n", + " # actually update the association file member\n", + " if os.path.exists(new_cat):\n", + " member['expname'] = new_cat\n", + " else:\n", + " print(f\"{new_cat} does not exist in the currenty working directory\")\n", + "\n", + " # write out the new association file\n", + " with open(asn, 'w', encoding='utf-8') as f:\n", + " json.dump(asn_data, f, ensure_ascii=False, indent=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "413eeb50-7e0a-4190-b999-f870b02fdc2e", + "metadata": {}, + "outputs": [], + "source": [ + "## double check that things still look ok:\n", + "asn_check = json.load(open(spec2_asns[0]))\n", + "for product in asn_check['products']:\n", + " for key, value in product.items():\n", + " if key == 'members':\n", + " print(f\"{key}:\")\n", + " for member in value:\n", + " print(f\" {member['expname']} : {member['exptype']}\")\n", + " else:\n", + " print(f\"{key}: {value}\")" ] }, { "cell_type": "markdown", - "id": "133f26bb", + "id": "5584e673-2907-4483-bb80-28bc16c11911", "metadata": {}, "source": [ - "## Run spec2 on all GR150R/C images using the provided association files" + "Alternatively, open the .json file in your favorite text editor and manually edit each of the catalog names" ] }, { "cell_type": "markdown", - "id": "18cd35b0", + "id": "d71ccaa3", "metadata": {}, "source": [ - "### Check an association file for level 2" + "\n", + "### Run spec2\n", + "\n", + "Like the image pipeline, we can supply custom parameters for steps in the pipeline.\n", + "\n", + "More information about everything that is run during the spec2 stage of the pipeline can be found [here](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_spec2.html).\n", + "\n", + "To start, we are only going to run spec2 on one association file because spec2 can take a while to run if there are many sources. We are saving the outputs to the directory we are currently in, which should be the `custom_spec2` directory made above." ] }, { "cell_type": "code", "execution_count": null, - "id": "353f52f4", + "id": "71e686d1-51b9-4548-8117-57a8815c9d39", "metadata": {}, "outputs": [], "source": [ - "asnfile = data_dir_asn_level2 + 'jw02079-o004_20230622t175524_spec2_00001_asn.json'\n", - "asn_data = json.load(open(asnfile))\n", - "print(asn_data['products'][0]['members'][0]['expname'])\n", - "asn_data" + "print(f'Calibrating: {spec2_asns[0]}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2d5b653", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "spec2 = Spec2Pipeline.call(spec2_asns[0], save_results=True)" ] }, { "cell_type": "markdown", - "id": "026e94a9", + "id": "c9dc3a2d-e33e-4f4d-88cb-c335e38bdcc8", "metadata": {}, "source": [ - "### Copy all needed files to the current directory" + "\n", + "### Examining the Outputs of Spec2\n", + "\n", + "The outputs of spec2 are `cal.fits` and `x1d.fits` files. Here we do a quick look into some important parts of these files.\n", + "- [cal further reading](https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/science_products.html#calibrated-data-cal-and-calints)\n", + "- [x1d further reading](https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/science_products.html#extracted-1-d-spectroscopic-data-x1d-and-x1dints)" ] }, { "cell_type": "code", "execution_count": null, - "id": "de5d11ec", + "id": "d84fe30e-3737-4886-ae51-28bfe21771e5", "metadata": {}, "outputs": [], "source": [ - "asn_spec2 = glob.glob(data_dir_asn_level2+'*spec2*')\n", - "for file in asn_spec2:\n", - " shutil.copy(file, './')" + "rate_file = asn_check['products'][0]['members'][0]['expname']\n", + "source_cat = asn_check['products'][0]['members'][2]['expname']\n", + "cal_file = rate_file.replace('rate', 'cal')\n", + "x1d_file = rate_file.replace('rate', 'x1d')" ] }, { "cell_type": "code", "execution_count": null, - "id": "dd56b572", + "id": "f0b56449-c61d-4330-9698-17818b95f7b7", "metadata": {}, "outputs": [], "source": [ - "rate_clear = listrate[np.where((listrate['FILTER']=='GR150R') | (listrate['FILTER']=='GR150C'))]['FILENAME']\n", - "for file in rate_clear:\n", - " shutil.copy(data_dir_in+file, './')" + "# open all of the files to look at\n", + "rate_hdu = fits.open(rate_file)\n", + "cal_hdu = fits.open(cal_file)\n", + "x1d_hdu = fits.open(x1d_file)\n", + "cat = Table.read(source_cat)\n", + "\n", + "# first look at how many sources we expect from the catalog\n", + "print(f'There are {len(cat)} sources identified in the current catalog.\\n')\n", + "\n", + "# then look at how long the cal and x1d files are for comparison\n", + "print(f'The x1d file has {len(x1d_hdu)} extensions & the cal file has {len(cal_hdu)} extensions')" ] }, { "cell_type": "code", "execution_count": null, - "id": "d765ef2c", + "id": "68ae6ca2-b804-4158-824f-0ed859d6a876", "metadata": {}, "outputs": [], "source": [ - "catalogfiles = glob.glob(data_dir_images+'*cat.ecsv')\n", - "for file in catalogfiles:\n", - " shutil.copy(file, './')" + "print(x1d_hdu.info())" ] }, { "cell_type": "code", "execution_count": null, - "id": "f2fa2925", + "id": "a0773e93-9baa-4d66-b181-a9623446159e", "metadata": {}, "outputs": [], "source": [ - "imagefiles = glob.glob(data_dir_images+'*i2d.fits')\n", - "for file in imagefiles:\n", - " shutil.copy(file, './')" + "print(cal_hdu.info())" + ] + }, + { + "cell_type": "markdown", + "id": "a0f6c961-dfba-4b12-a7ee-cb61623308dd", + "metadata": {}, + "source": [ + "Note that the 0th and final extension in each file do not contain science data, but the remaining extensions correspond to each source. The `x1d` file contains the extracted spectrum for each source, while the `cal` file contains the 2D cutout information in seven extensions for each source (SCI, DQ, ERR, WAVELENGTH, VAR_POISSON, VAR_RNOISE, VAR_FLAT).\n", + "\n", + "This in part is why it is so important to have a refined source catalog. If there are sources that are not useful for your research, there is no reason to create a cutout and extract them." ] }, { "cell_type": "code", "execution_count": null, - "id": "531a997b", + "id": "1fa8d0c6-ea17-475a-ab74-2a40fba25670", "metadata": {}, "outputs": [], "source": [ - "segfiles = glob.glob(data_dir_images+'*segm.fits')\n", - "for file in segfiles:\n", - " shutil.copy(file, './')" + "# a quick look at the columns also available in the x1d file\n", + "print(x1d_hdu[1].data.columns)" ] }, { "cell_type": "markdown", - "id": "d71ccaa3", + "id": "2e1a31a6-fbba-4ca5-920a-ac59014c8495", + "metadata": {}, + "source": [ + "\n", + "## Explore Spec2 Data Further" + ] + }, + { + "cell_type": "markdown", + "id": "ba3ce4c1-64c0-4fcb-a274-b1003fade978", "metadata": {}, "source": [ - "## Run spec2" + "\n", + "### Find a Source in the Spec2 Data\n", + "\n", + "Each extension of the cal and x1d files has a source ID in the header. With newer versions of the JWST pipeline, these values should match with the values in the source catalog." ] }, { "cell_type": "code", "execution_count": null, - "id": "b114db48", + "id": "d7504a2f-74b9-4f02-a760-79a99a1b80be", "metadata": {}, "outputs": [], "source": [ - "asn_spec2_local = glob.glob('./jw02079-o004_*_spec2_*_asn.json')\n", - "asn_spec2_local" + "def find_source_ext(x1d_hdu, cal_hdu, source_id, info=True):\n", + " # x1d extension first\n", + " x1d_source_ids = np.array([x1d_hdu[ext].header['SOURCEID'] for ext in range(len(x1d_hdu))[1:-1]]) # cut off the 0th and last data extensions\n", + " wh_x1d = np.where(x1d_source_ids == source_id)[0][0] + 1 # need to add 1 for the primary header\n", + " if info:\n", + " print(f\"Extension {wh_x1d} in {x1d_hdu[0].header['FILENAME']} contains the data for source {source_id} from our catalog\")\n", + " \n", + " # look for cal extension, too, but only in the SCI extension; \n", + " # fill in with a source ID of -999 for all other extensions to get the right extension value\n", + " cal_source_ids = np.array([cal_hdu[ext].header['SOURCEID'] if cal_hdu[ext].header['EXTNAME'] == 'SCI' else -999 for ext in range(len(cal_hdu))[1:-1] ]) \n", + " wh_cal = np.where(cal_source_ids == source_id)[0][0] + 1 # need to add 1 for the primary header\n", + " if info:\n", + " print(f\"Extension {wh_cal} in {cal_hdu[0].header['FILENAME']} contains the data for source {source_id} from our catalog\")\n", + "\n", + " return wh_x1d, wh_cal" ] }, { "cell_type": "code", "execution_count": null, - "id": "b2d5b653", + "id": "df474283-d660-4b5a-a4f8-9e5a6b187c30", + "metadata": {}, + "outputs": [], + "source": [ + "# Let's look for source 155 that we identified in the previous notebook.\n", + "source_id = 155\n", + "wh_x1d_155, wh_cal_155 = find_source_ext(x1d_hdu, cal_hdu, source_id)\n", + "\n", + "x1d_data_155 = x1d_hdu[wh_x1d_155].data \n", + "cal_data_155 = cal_hdu[wh_cal_155].data" + ] + }, + { + "cell_type": "markdown", + "id": "a362fa9f-ae33-4dcc-954c-471a80b41d37", + "metadata": {}, + "source": [ + "First, let's look at the extraction box as it appears on the rate image. This will help us get a global feel for how much we can trust the pipeline's extraction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ddc84d02-8b67-440b-9380-53d5a27a5349", + "metadata": {}, + "outputs": [], + "source": [ + "# fill in the nan values from the bad pixels with zero to make it easier to look at\n", + "rate_data = rate_hdu[1].data\n", + "rate_data[np.isnan(rate_data)] = 0\n", + "\n", + "# extraction box parameters from the header of the cal data:\n", + "cal_header = cal_hdu[wh_cal_155].header\n", + "sx_left = cal_header['SLTSTRT1']\n", + "swidth = cal_header['SLTSIZE1']\n", + "sx_right = cal_header['SLTSTRT1'] + swidth\n", + "sy_bottom = cal_header['SLTSTRT2']\n", + "sheight = cal_header['SLTSIZE2']\n", + "sy_top = cal_header['SLTSTRT2'] + sheight\n", + "\n", + "# plot the rate file and the extraction box\n", + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 5))\n", + "ax1.imshow(rate_data, origin='lower', vmin=0, vmax=2, aspect='auto')\n", + "ax2.imshow(rate_data, origin='lower', vmin=0, vmax=2, aspect='auto')\n", + "\n", + "rectangle = patches.Rectangle((sx_left, sy_bottom), swidth, sheight, edgecolor='darkorange', facecolor=\"None\", linewidth=1)\n", + "ax1.add_patch(rectangle)\n", + "ax1.set_title(rate_file)\n", + "\n", + "rectangle2 = patches.Rectangle((sx_left, sy_bottom), swidth, sheight, edgecolor='darkorange', facecolor=\"None\", linewidth=2)\n", + "ax2.add_patch(rectangle2)\n", + "ax2.set_xlim(sx_left-30, sx_right+30)\n", + "ax2.set_ylim(sy_bottom-30, sy_top+30)\n", + "ax2.set_title(f'Source {source_id} Zoom in')\n", + "\n", + "plt.suptitle(f\"{cal_hdu[0].header['FILTER']} {cal_hdu[0].header['PUPIL']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87f80f3f-b2c9-4496-a1e9-0b8005b49f7c", + "metadata": {}, + "outputs": [], + "source": [ + "def plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file):\n", + " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 5))\n", + " \n", + " # plot the cal image\n", + " ax1.imshow(cal_data, origin='lower', vmin=0, vmax=np.nanmean(cal_data), aspect='auto')\n", + " ax1.set_title(os.path.basename(cal_file))\n", + " \n", + " # plot the spectrum\n", + " wave = x1d_data['WAVELENGTH'].ravel()\n", + " flux = x1d_data['FLUX'].ravel()\n", + "\n", + " ax2.plot(wave, flux)\n", + " ax2.set_title(os.path.basename(x1d_file))\n", + "\n", + " edge_buffer = int(len(flux) * .25)\n", + " max_flux = np.nanmax(flux[edge_buffer:edge_buffer*-1])\n", + " ax2.set_ylim(0, max_flux+(max_flux*0.1)) # cutting the flux of the edges & adding 10% buffer to the limits\n", + " \n", + " plt.suptitle(f\"{fits.getval(cal_file, 'FILTER')} {fits.getval(cal_file, 'PUPIL')}\")\n", + "\n", + "plot_cutout_and_spectrum(cal_data_155, x1d_data_155, cal_file, x1d_file)" + ] + }, + { + "cell_type": "markdown", + "id": "b4e0c7f9-2400-4324-a831-d614bb51c22f", + "metadata": {}, + "source": [ + "\n", + "### Limit the Number of Extracted Sources\n", + "\n", + "Because it takes so long to extract so many sources, let's see if we can pair down the number of sources being extracted. We'll do this with parameter inputs (for bright point sources) and with a further refined source catalog." + ] + }, + { + "cell_type": "markdown", + "id": "c815381d-b50b-404a-9941-e21696e20542", + "metadata": {}, + "source": [ + "#### Limit Extraction Using Parameter Inputs\n", + "For this calibration, we are explicitly calling out parameters the following step:\n", + "- `extract_2d` where we are setting `wfss_nbright` which limits the number of bright sources that are extracted and `wfss_mmag_extract` which sets a limit on the faintest magnitude we want extracted. [Further Reading](https://jwst-pipeline.readthedocs.io/en/latest/jwst/extract_2d/main.html#nircam-and-niriss-wfss)\n", + "\n", + "In this case, we'll limit the extractions to only the 10 brightest objects." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9fb2d1af-e529-497d-92db-390d17905eb9", + "metadata": {}, + "outputs": [], + "source": [ + "param_outdir = 'parameter_input_calibrated'\n", + "if not os.path.exists(param_outdir):\n", + " os.mkdir(param_outdir)\n", + " \n", + "spec2 = Spec2Pipeline.call(spec2_asns[0],\n", + " steps={'extract_2d':{'wfss_nbright' : 10,},},\n", + " save_results=True,\n", + " output_dir=param_outdir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c2e9d47-0e79-4005-8aa6-d3eaf1264a33", + "metadata": {}, + "outputs": [], + "source": [ + "# open the x1d to examine how many sources were extracted\n", + "x1ds = glob.glob(os.path.join(param_outdir, '*x1d.fits*'))\n", + "with fits.open(x1ds[0]) as temp_x1d:\n", + " print(f'The x1d file has {len(temp_x1d)-2} extracted sources')" + ] + }, + { + "cell_type": "markdown", + "id": "43e624bc-ea7b-41e5-a923-8c62374c9b13", + "metadata": {}, + "source": [ + "#### Limit Extraction Using the Source Catalog\n", + "In the last notebook we limited the catalog a couple of different ways. Here, let's limit the catalog to a specific magnitude range, and then use that new catalog to extract the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1cbc550-6f54-4f2b-b887-617661fabd07", + "metadata": {}, + "outputs": [], + "source": [ + "# look at the minimum and maximum magnitudes\n", + "mags = cat['isophotal_vegamag']\n", + "min_vegmag = mags.min()\n", + "max_vegmag = mags.max()\n", + "print(f\"Magnitude range: {min_vmag} - {max_vmag}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cd470d5-f0f7-4f81-adf9-52e05f7dadfa", + "metadata": {}, + "outputs": [], + "source": [ + "# find the catalog for how many stars are between a specific magnitude \n", + "min_mag = 20.5\n", + "max_mag = 21\n", + "mag_cat = cat[(cat['isophotal_vegamag'] >= min_mag) & (cat['isophotal_vegamag'] <= max_mag)]\n", + "\n", + "mag_cat['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3670bd3b-00c7-40cb-906d-b63e44c044d9", + "metadata": {}, + "outputs": [], + "source": [ + "new_cat = Table(mag_cat) # turn the row instance into a dataframe again\n", + "\n", + "# save the new catalog with a unique name\n", + "new_cat_name = source_cat.replace('cat.ecsv', f'mag-limit_cat.ecsv')\n", + "new_cat.write(new_cat_name, overwrite=True)\n", + "print('Saved:', new_cat_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e919b35-3558-4671-99ca-9f184f59a707", + "metadata": {}, + "outputs": [], + "source": [ + "print(list(mag_cat['label']))\n", + "cat.loc[list(mag_cat['label'])]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afc40e15-a182-4440-8d6a-920c0ea8f526", + "metadata": {}, + "outputs": [], + "source": [ + "# apply the same conditions to all of the source catalogs now if you're happy with your limits\n", + "min_mag = 20.5\n", + "max_mag = 21\n", + "\n", + "mag_source_ext = f'mag-limit_cat.ecsv'\n", + "\n", + "for cat_name in glob.glob('*source-match_cat.ecsv'):\n", + "\n", + " if cat_name == source_cat:\n", + " # the base one has already been saved\n", + " continue\n", + "\n", + " # pandas can't handle the .ecsv format, but we astropy table doesn't have the \"isin\" feature\n", + " cat = Table.read(cat_name)\n", + " cat.add_index('label')\n", + " new_cat = cat.loc[list(mag_cat['label'])]\n", + "\n", + " print(repr(new_cat['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']))\n", + " # save the new catalog with a unique name\n", + " new_cat_name = cat_name.replace('cat.ecsv', mag_source_ext)\n", + " new_cat.write(new_cat_name, overwrite=True)\n", + " print('Saved:', new_cat_name)\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "570bc05e-f84f-42ab-9067-ea9dd072de75", + "metadata": {}, + "source": [ + "Once the new catalogs have been made, we have to update the association files to use the new catalogs. **Note: the association files will need to be updated again if you want to calibrate again with the previous source catalogs**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ced9ad72-78d6-4718-8de4-1e5e108db6dc", + "metadata": {}, + "outputs": [], + "source": [ + "new_source_ext = mag_source_ext\n", + "\n", + "# loop through all of the spec2 association files in the current directory\n", + "for asn in spec2_asns:\n", + " asn_data = json.load(open(asn))\n", + " for product in asn_data['products']: # for every product, check the members \n", + " for member in product['members']: # there are multiple members per product\n", + " if member['exptype'] == 'sourcecat':\n", + " cat_in_asn = member['expname']\n", + " # check that we haven't already updated the source catalog name\n", + " if new_source_ext not in cat_in_asn:\n", + " new_cat = cat_in_asn.replace('cat.ecsv', new_source_ext)\n", + " # actually update the association file member\n", + " if os.path.exists(new_cat):\n", + " member['expname'] = new_cat\n", + " else:\n", + " print(f\"{new_cat} does not exist in the currenty working directory\")\n", + "\n", + " # write out the new association file\n", + " with open(asn, 'w', encoding='utf-8') as f:\n", + " json.dump(asn_data, f, ensure_ascii=False, indent=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e839a1a-5c23-4185-8033-e43d0b367037", "metadata": { "scrolled": true }, "outputs": [], "source": [ - "for file in asn_spec2_local:\n", - " print(file)\n", - " spec2 = Spec2Pipeline() \n", - " result = spec2.call(file, \n", - " steps={'bkg_subtract':{'skip':False},\n", - " 'extract_2d':{'wfss_nbright':500,\n", - " 'wfss_extract_half_height': 25,\n", - " },\n", - " 'photom':{'skip':False},\n", - " },\n", - " save_results=True,\n", - " output_dir=data_dir_out_spec2)\n" + "## double check that the source catalog has been changed\n", + "for spec2_asn in spec2_asns:\n", + " asn_check = json.load(open(spec2_asn))\n", + " for product in asn_check['products']:\n", + " for key, value in product.items():\n", + " if key == 'members':\n", + " #print(f\"{key}:\")\n", + " for member in value:\n", + " if member['exptype'] == 'sourcecat':\n", + " print(f\" {member['exptype']}: {member['expname']}\")\n", + " else:\n", + " print(f\"{key}: {value}\")" ] }, { "cell_type": "markdown", - "id": "b06abd18", + "id": "c5dd5d4c-5ef0-4690-a4da-a795c295a313", "metadata": {}, "source": [ - "### Delete files not needed anymore" + "Now when we calibrate everything, it should take a lot less time because there are a limited number of sources." ] }, { "cell_type": "code", "execution_count": null, - "id": "a8c142f2", + "id": "6b7dad07-7d44-4ebe-841d-3f686b9df016", + "metadata": {}, + "outputs": [], + "source": [ + "## calibrate with the new source catalog\n", + "source_outdir = 'new_catalog_calibrated'\n", + "if not os.path.exists(source_outdir):\n", + " os.mkdir(source_outdir)\n", + "\n", + "for spec2_asn in spec2_asns:\n", + " spec2 = Spec2Pipeline.call(spec2_asn, save_results=True, output_dir=source_outdir)" + ] + }, + { + "cell_type": "markdown", + "id": "d03794fa-eee5-402a-af65-96378351275d", "metadata": {}, + "source": [ + "\n", + "### Final Visualization\n", + "\n", + "Now that everything has been calibrated, it's useful to look at all of the extracted files for at least one of the sources. \n", + "\n", + "Continue to explore further, including using the [spec3 stage](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_spec3.html) of the pipeline!" + ] + }, + { + "cell_type": "markdown", + "id": "83e7c35f-cf6e-4d30-be38-5b16c80c1487", + "metadata": {}, + "source": [ + "##### Look at all of the Files for a Single Source" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94aec550-ea1f-4215-8965-bcd2ac408bfd", + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "dir_to_be_del = './files_to_be_deleted'\n", - "if not os.path.exists(dir_to_be_del):\n", - " os.mkdir(dir_to_be_del)\n", + "#### Explore the new Data\n", + "x1ds = np.sort(glob.glob(os.path.join(source_outdir, \"*x1d.fits\")))\n", + "\n", + "# get a list of all of the source IDs from the first file to look at for this example\n", + "sources = [fits.getval(x1ds[0], 'SOURCEID', ext=ext) for ext in range(len(fits.open(x1ds[0])))[1:-1]]\n", + "source_id = sources[0]\n", + "\n", + "# plot each x1d/cal file\n", + "for i, x1d_file in enumerate(x1ds):\n", + " cal_file = x1d_file.replace('x1d.fits', 'cal.fits')\n", + " with fits.open(x1d_file) as x1d_hdu, fits.open(cal_file) as cal_hdu:\n", + "\n", + " try:\n", + " wh_x1d, wh_cal = find_source_ext(x1d_hdu, cal_hdu, source_id, info=False)\n", + " except IndexError:\n", + " # this means the source isn't in this observation\n", + " continue\n", + "\n", + " x1d_data = x1d_hdu[wh_x1d].data \n", + " cal_data = cal_hdu[wh_cal].data\n", + "\n", + " plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file)" + ] + }, + { + "cell_type": "markdown", + "id": "123375a9-4660-4205-9128-4b077bfc06dc", + "metadata": {}, + "source": [ + "##### Look at all of the sources for a single file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa3b5777-4368-41fa-942f-c3a6f003d1a3", + "metadata": {}, + "outputs": [], + "source": [ + "x1d_file = x1ds[0]\n", + "cal_file = x1d_file.replace('x1d.fits', 'cal.fits')\n", + "with fits.open(x1d_file) as x1d_hdu, fits.open(cal_file) as cal_hdu:\n", + "\n", + " for ext in range(len(x1d_hdu))[1:-1]:\n", + "\n", + " source_id = x1d_hdu[ext].header['SOURCEID']\n", + " print(source_id)\n", + " try:\n", + " wh_x1d, wh_cal = find_source_ext(x1d_hdu, cal_hdu, source_id, info=False)\n", + " except IndexError:\n", + " # this means the source isn't in this observation\n", + " continue\n", " \n", - "for file in asn_spec2_local:\n", - " shutil.move(file, dir_to_be_del)\n", + " x1d_data = x1d_hdu[wh_x1d].data \n", + " cal_data = cal_hdu[wh_cal].data\n", " \n", - "for file in glob.glob('./*rate.fits'):\n", - " shutil.move(file, dir_to_be_del)\n", + " plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file)" + ] + }, + { + "cell_type": "markdown", + "id": "13a0285f-edb3-4796-8377-774a65d02ecb", + "metadata": {}, + "source": [ + "##### Visualize where the extracted sources are on the dispersed image, and how that compares to the direct image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d87810b9-efc5-4554-a2c0-11b5338c7b8b", + "metadata": {}, + "outputs": [], + "source": [ + "x1d_file = x1ds[0]\n", + "cal_file = x1d_file.replace('x1d.fits', 'cal.fits')\n", + "rate_file = os.path.basename(x1d_file.replace('x1d.fits', 'rate.fits'))\n", + "\n", + "# fill in the nan values from the bad pixels with zero to make it easier to look at\n", + "with fits.open(rate_file) as rate_hdu:\n", + " rate_data = rate_hdu[1].data\n", + "rate_data[np.isnan(rate_data)] = 0\n", + "\n", + "# plot the rate file and the extraction box\n", + "fig, ax = plt.subplots(1, 1, figsize=(10, 10))\n", + "ax.imshow(rate_data, origin='lower', vmin=0, vmax=np.nanmean(cal_data)*0.2, aspect='auto')\n", + "\n", + "with fits.open(x1d_file) as x1d_hdu, fits.open(cal_file) as cal_hdu:\n", + "\n", + " for ext in range(len(x1d_hdu))[1:-1]:\n", + "\n", + " source_id = x1d_hdu[ext].header['SOURCEID']\n", + "\n", + " try:\n", + " wh_x1d, wh_cal = find_source_ext(x1d_hdu, cal_hdu, source_id, info=False)\n", + " except IndexError:\n", + " # this means the source isn't in this observation\n", + " continue\n", + " \n", + " x1d_data = x1d_hdu[wh_x1d].data \n", + " cal_data = cal_hdu[wh_cal].data\n", + " \n", + " # extraction box parameters from the header of the cal data:\n", + " cal_header = cal_hdu[wh_cal].header\n", + " sx_left = cal_header['SLTSTRT1']\n", + " swidth = cal_header['SLTSIZE1']\n", + " sx_right = cal_header['SLTSTRT1'] + swidth\n", + " sy_bottom = cal_header['SLTSTRT2']\n", + " sheight = cal_header['SLTSIZE2']\n", + " sy_top = cal_header['SLTSTRT2'] + sheight\n", + " \n", + " rectangle = patches.Rectangle((sx_left, sy_bottom), swidth, sheight, edgecolor='darkorange', facecolor=\"None\", linewidth=1)\n", + " ax.add_patch(rectangle)\n", + " ax.text(sx_left, sy_top+10, source_id, fontsize=14, color='darkorange')\n", + " \n", + " ax.set_title(f\"{os.path.basename(x1d_file)}: {cal_hdu[0].header['FILTER']} {cal_hdu[0].header['PUPIL']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d1cacd1-44cb-41f9-9ead-245d2e61637a", + "metadata": {}, + "outputs": [], + "source": [ + "asn_data = json.load(open(fits.getval(x1d_file, 'ASNTABLE')))\n", + "i2d_name = asn_data['products'][0]['members'][1]['expname']\n", + "cat_name = asn_data['products'][0]['members'][2]['expname']\n", + "\n", + "fig, ax = plt.subplots(1, 1, figsize=(10, 10))\n", + "with fits.open(os.path.join('../../', custom_run_image3, i2d_name)) as i2d:\n", + " ax.imshow(i2d[1].data, origin='lower', aspect='auto', vmin=0, vmax=np.nanmax(i2d[1].data)*0.01)\n", + "\n", + "# also plot the associated catalog\n", + "cat = Table.read(cat_name)\n", + "extended_sources = cat[cat['is_extended'] == True]\n", + "point_sources = cat[cat['is_extended'] == False]\n", "\n", - "for file in glob.glob('./*niriss_clear*'):\n", - " shutil.move(file, dir_to_be_del)\n", + "for color, sources in zip(['darkorange', 'black'], [extended_sources, point_sources]):\n", + " # plotting the sources\n", + " ax.scatter(sources['xcentroid'], sources['ycentroid'], s=40, facecolors='None', edgecolors=color, alpha=0.9, lw=2)\n", "\n", - "shutil.rmtree(dir_to_be_del)" + " # adding source labels \n", + " for i, source_num in enumerate(sources['label']):\n", + " ax.annotate(source_num, \n", + " (sources['xcentroid'][i]+0.5, sources['ycentroid'][i]+0.5), \n", + " fontsize=14,\n", + " color=color)\n", + " " ] }, { "cell_type": "code", "execution_count": null, - "id": "64a3bba8", + "id": "b196ecdb-4a07-4534-8f15-7dc0f27824af", "metadata": {}, "outputs": [], "source": [] @@ -300,7 +1050,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.11.7" } }, "nbformat": 4, From 799cbb8aead5f3c6186ebdc79ec24ab75b5f8e6b Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 1 Feb 2024 18:50:54 -0500 Subject: [PATCH 25/62] Renaming order of notebooks for spec2 --- .../{06_run_pipeline_spec2.ipynb => 02_niriss_wfss_spec2.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename notebooks/NIRISS_WFSS_advanced/{06_run_pipeline_spec2.ipynb => 02_niriss_wfss_spec2.ipynb} (100%) diff --git a/notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb similarity index 100% rename from notebooks/NIRISS_WFSS_advanced/06_run_pipeline_spec2.ipynb rename to notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb From 951f63583136efa721956c93bf0ab3bf0a6c64e4 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 2 Feb 2024 12:17:33 -0500 Subject: [PATCH 26/62] cleanup of spec2 --- .../02_niriss_wfss_spec2.ipynb | 196 +++++++++++------- 1 file changed, 126 insertions(+), 70 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index d96c1f686..95bb5964d 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -7,7 +7,7 @@ "source": [ "# Run spec2 pipeline\n", "\n", - "The dispersed images for WFSS will be run through the spec2 pipeline after the image2 and image3 pipelines have been run on the corresponding direct images. As mentioned in the previous notebook, it is extremely helpful for this step if you have a source catalog that only includes the sources that you would like to look at. Any additional sources will take extra time to calibrate.\n", + "The dispersed images for WFSS will be run through the [spec2 pipeline](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_spec2.html) after the image2 and image3 pipelines have been run on the corresponding direct images. As mentioned in the previous notebook, it is extremely helpful for this step if you have a source catalog that only includes the sources that you would like to look at. Any additional sources will take extra time to calibrate.\n", "\n", "**Use case**: After creating a custom source catalog, spec2 should be run on the dispersed WFSS images.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", @@ -110,7 +110,7 @@ "id": "d218c8e0-7a25-4688-b904-c4225a66a8ca", "metadata": {}, "source": [ - "Open up our data information file which will make parsing the rate files to copy over easier." + "Open up our data information file which will make parsing the rate files to copy over easier as we only need the dispersed images, i.e. those observed with GR150R or GR150C." ] }, { @@ -212,6 +212,24 @@ "print('Now in:', new_cwd)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "84915ddb-15a0-491b-ada1-965112ac9311", + "metadata": {}, + "outputs": [], + "source": [ + "# directories for calibrating later\n", + "source_outdir = 'new_catalog_calibrated'\n", + "if not os.path.exists(source_outdir):\n", + " os.mkdir(source_outdir)\n", + "\n", + "\n", + "param_outdir = 'parameter_input_calibrated'\n", + "if not os.path.exists(param_outdir):\n", + " os.mkdir(param_outdir)" + ] + }, { "cell_type": "markdown", "id": "e0a176fd-094d-44e5-b860-5b9f730e0898", @@ -373,7 +391,8 @@ "metadata": {}, "outputs": [], "source": [ - "print(f'Calibrating: {spec2_asns[0]}')" + "test_asn = spec2_asns[0]\n", + "print(f'Calibrating: {test_asn}')" ] }, { @@ -385,7 +404,14 @@ }, "outputs": [], "source": [ - "spec2 = Spec2Pipeline.call(spec2_asns[0], save_results=True)" + "# check if the calibrated file already exists\n", + "asn_data = json.load(open(test_asn))\n", + "x1d_file = f\"{asn_data['products'][0]['name']}_x1d.fits\"\n", + "\n", + "if os.path.exists(x1d_file):\n", + " print(x1d_file, ': x1d file already exists.')\n", + "else:\n", + " spec2 = Spec2Pipeline.call(test_asn, save_results=True)" ] }, { @@ -408,8 +434,9 @@ "metadata": {}, "outputs": [], "source": [ - "rate_file = asn_check['products'][0]['members'][0]['expname']\n", - "source_cat = asn_check['products'][0]['members'][2]['expname']\n", + "asn_example = json.load(open(spec2_asns[0]))\n", + "rate_file = asn_example['products'][0]['members'][0]['expname']\n", + "source_cat = asn_example['products'][0]['members'][2]['expname']\n", "cal_file = rate_file.replace('rate', 'cal')\n", "x1d_file = rate_file.replace('rate', 'x1d')" ] @@ -434,6 +461,18 @@ "print(f'The x1d file has {len(x1d_hdu)} extensions & the cal file has {len(cal_hdu)} extensions')" ] }, + { + "cell_type": "markdown", + "id": "a0f6c961-dfba-4b12-a7ee-cb61623308dd", + "metadata": {}, + "source": [ + "Note that the 0th and final extension in each file do not contain science data, but the remaining extensions correspond to each source. The `x1d` file contains the extracted spectrum for each source, while the `cal` file contains the 2D cutout information in seven extensions for each source (SCI, DQ, ERR, WAVELENGTH, VAR_POISSON, VAR_RNOISE, VAR_FLAT).\n", + "\n", + "This in part is why it is so important to have a refined source catalog. If there are sources that are not useful for your research, there is no reason to create a cutout and extract them.\n", + "\n", + "Notice that there are more sources than there are extensions in the files. This is because the pipeline defaults to only extracting the 100 brightest sources. To change this behavior, supply the pipeline with the paramter `wfss_nbright`, which we do [below](#limit_source)." + ] + }, { "cell_type": "code", "execution_count": null, @@ -456,12 +495,10 @@ }, { "cell_type": "markdown", - "id": "a0f6c961-dfba-4b12-a7ee-cb61623308dd", + "id": "52473e1d-c5b4-490d-b2e9-e0d049897c7b", "metadata": {}, "source": [ - "Note that the 0th and final extension in each file do not contain science data, but the remaining extensions correspond to each source. The `x1d` file contains the extracted spectrum for each source, while the `cal` file contains the 2D cutout information in seven extensions for each source (SCI, DQ, ERR, WAVELENGTH, VAR_POISSON, VAR_RNOISE, VAR_FLAT).\n", - "\n", - "This in part is why it is so important to have a refined source catalog. If there are sources that are not useful for your research, there is no reason to create a cutout and extract them." + "The `x1d` file is a BinTable, so there are additional columns contained inside each of the data extensions. [x1d further reading](https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/science_products.html#extracted-1-d-spectroscopic-data-x1d-and-x1dints) goes through what is contained in each column, but we can also take a quick look by looking at one of the data columns." ] }, { @@ -605,8 +642,16 @@ " max_flux = np.nanmax(flux[edge_buffer:edge_buffer*-1])\n", " ax2.set_ylim(0, max_flux+(max_flux*0.1)) # cutting the flux of the edges & adding 10% buffer to the limits\n", " \n", - " plt.suptitle(f\"{fits.getval(cal_file, 'FILTER')} {fits.getval(cal_file, 'PUPIL')}\")\n", - "\n", + " plt.suptitle(f\"{fits.getval(cal_file, 'FILTER')} {fits.getval(cal_file, 'PUPIL')}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca35719a-8eb7-4ba3-87c8-82cb067da4b8", + "metadata": {}, + "outputs": [], + "source": [ "plot_cutout_and_spectrum(cal_data_155, x1d_data_155, cal_file, x1d_file)" ] }, @@ -640,14 +685,17 @@ "metadata": {}, "outputs": [], "source": [ - "param_outdir = 'parameter_input_calibrated'\n", - "if not os.path.exists(param_outdir):\n", - " os.mkdir(param_outdir)\n", - " \n", - "spec2 = Spec2Pipeline.call(spec2_asns[0],\n", - " steps={'extract_2d':{'wfss_nbright' : 10,},},\n", - " save_results=True,\n", - " output_dir=param_outdir)" + "# check if the calibrated file already exists\n", + "asn_data = json.load(open(spec2_asns[0]))\n", + "x1d_file = os.path.join(param_outdir, f\"{asn_data['products'][0]['name']}_x1d.fits\")\n", + "\n", + "if os.path.exists(x1d_file):\n", + " print(x1d_file, ': x1d file already exists.')\n", + "else:\n", + " spec2 = Spec2Pipeline.call(spec2_asns[0],\n", + " steps={'extract_2d':{'wfss_nbright' : 10,},},\n", + " save_results=True,\n", + " output_dir=param_outdir)" ] }, { @@ -683,7 +731,7 @@ "mags = cat['isophotal_vegamag']\n", "min_vegmag = mags.min()\n", "max_vegmag = mags.max()\n", - "print(f\"Magnitude range: {min_vmag} - {max_vmag}\")" + "print(f\"Magnitude range: {min_vegmag} - {max_vegmag}\")" ] }, { @@ -708,23 +756,22 @@ "metadata": {}, "outputs": [], "source": [ + "mag_source_ext = f'mag-limit_cat.ecsv'\n", + "\n", "new_cat = Table(mag_cat) # turn the row instance into a dataframe again\n", "\n", "# save the new catalog with a unique name\n", - "new_cat_name = source_cat.replace('cat.ecsv', f'mag-limit_cat.ecsv')\n", + "new_cat_name = source_cat.replace('cat.ecsv', mag_source_ext)\n", "new_cat.write(new_cat_name, overwrite=True)\n", "print('Saved:', new_cat_name)" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "8e919b35-3558-4671-99ca-9f184f59a707", + "cell_type": "markdown", + "id": "6c2f2072-b19d-4aab-92d6-2e0570bda9d8", "metadata": {}, - "outputs": [], "source": [ - "print(list(mag_cat['label']))\n", - "cat.loc[list(mag_cat['label'])]" + "Once we have a source catalog that we've limited to specific sources, let's match the remaining catalogs to those sources" ] }, { @@ -734,24 +781,20 @@ "metadata": {}, "outputs": [], "source": [ - "# apply the same conditions to all of the source catalogs now if you're happy with your limits\n", - "min_mag = 20.5\n", - "max_mag = 21\n", - "\n", - "mag_source_ext = f'mag-limit_cat.ecsv'\n", - "\n", "for cat_name in glob.glob('*source-match_cat.ecsv'):\n", "\n", " if cat_name == source_cat:\n", " # the base one has already been saved\n", " continue\n", "\n", - " # pandas can't handle the .ecsv format, but we astropy table doesn't have the \"isin\" feature\n", + " # match the source IDs between the current catalog and the base catalog above\n", " cat = Table.read(cat_name)\n", " cat.add_index('label')\n", " new_cat = cat.loc[list(mag_cat['label'])]\n", "\n", + " # check to ensure the sources are all there\n", " print(repr(new_cat['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']))\n", + " \n", " # save the new catalog with a unique name\n", " new_cat_name = cat_name.replace('cat.ecsv', mag_source_ext)\n", " new_cat.write(new_cat_name, overwrite=True)\n", @@ -828,6 +871,16 @@ "Now when we calibrate everything, it should take a lot less time because there are a limited number of sources." ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "f60f8622-857c-43de-8f18-f251e8e85aa4", + "metadata": {}, + "outputs": [], + "source": [ + "ls new_catalog_calibrated/jw02079004001_03101_00001_nis_x1d.fits" + ] + }, { "cell_type": "code", "execution_count": null, @@ -836,12 +889,15 @@ "outputs": [], "source": [ "## calibrate with the new source catalog\n", - "source_outdir = 'new_catalog_calibrated'\n", - "if not os.path.exists(source_outdir):\n", - " os.mkdir(source_outdir)\n", - "\n", "for spec2_asn in spec2_asns:\n", - " spec2 = Spec2Pipeline.call(spec2_asn, save_results=True, output_dir=source_outdir)" + " # check if the calibrated file already exists\n", + " asn_data = json.load(open(spec2_asn))\n", + " x1d_file = os.path.join(source_outdir, f\"{asn_data['products'][0]['name']}_x1d.fits\")\n", + " \n", + " if os.path.exists(x1d_file):\n", + " print(x1d_file, ': x1d file already exists.')\n", + " else:\n", + " spec2 = Spec2Pipeline.call(spec2_asn, save_results=True, output_dir=source_outdir)" ] }, { @@ -879,7 +935,7 @@ "\n", "# get a list of all of the source IDs from the first file to look at for this example\n", "sources = [fits.getval(x1ds[0], 'SOURCEID', ext=ext) for ext in range(len(fits.open(x1ds[0])))[1:-1]]\n", - "source_id = sources[0]\n", + "source_id = 245\n", "\n", "# plot each x1d/cal file\n", "for i, x1d_file in enumerate(x1ds):\n", @@ -910,7 +966,9 @@ "cell_type": "code", "execution_count": null, "id": "aa3b5777-4368-41fa-942f-c3a6f003d1a3", - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "x1d_file = x1ds[0]\n", @@ -948,6 +1006,12 @@ "metadata": {}, "outputs": [], "source": [ + "# setting up the figure\n", + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 7))\n", + "\n", + "## grism data\n", + "\n", + "# for the dispersed image plot\n", "x1d_file = x1ds[0]\n", "cal_file = x1d_file.replace('x1d.fits', 'cal.fits')\n", "rate_file = os.path.basename(x1d_file.replace('x1d.fits', 'rate.fits'))\n", @@ -958,8 +1022,7 @@ "rate_data[np.isnan(rate_data)] = 0\n", "\n", "# plot the rate file and the extraction box\n", - "fig, ax = plt.subplots(1, 1, figsize=(10, 10))\n", - "ax.imshow(rate_data, origin='lower', vmin=0, vmax=np.nanmean(cal_data)*0.2, aspect='auto')\n", + "ax1.imshow(rate_data, origin='lower', vmin=0, vmax=np.nanmean(cal_data)*0.2, aspect='auto')\n", "\n", "with fits.open(x1d_file) as x1d_hdu, fits.open(cal_file) as cal_hdu:\n", "\n", @@ -986,26 +1049,18 @@ " sy_top = cal_header['SLTSTRT2'] + sheight\n", " \n", " rectangle = patches.Rectangle((sx_left, sy_bottom), swidth, sheight, edgecolor='darkorange', facecolor=\"None\", linewidth=1)\n", - " ax.add_patch(rectangle)\n", - " ax.text(sx_left, sy_top+10, source_id, fontsize=14, color='darkorange')\n", + " ax1.add_patch(rectangle)\n", + " ax1.text(sx_left, sy_top+10, source_id, fontsize=14, color='darkorange')\n", " \n", - " ax.set_title(f\"{os.path.basename(x1d_file)}: {cal_hdu[0].header['FILTER']} {cal_hdu[0].header['PUPIL']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8d1cacd1-44cb-41f9-9ead-245d2e61637a", - "metadata": {}, - "outputs": [], - "source": [ + " ax1.set_title(f\"{os.path.basename(x1d_file).split('_nis')[0]}: {cal_hdu[0].header['FILTER']} {cal_hdu[0].header['PUPIL']}\")\n", + "\n", + "## imaging data\n", "asn_data = json.load(open(fits.getval(x1d_file, 'ASNTABLE')))\n", "i2d_name = asn_data['products'][0]['members'][1]['expname']\n", "cat_name = asn_data['products'][0]['members'][2]['expname']\n", - "\n", - "fig, ax = plt.subplots(1, 1, figsize=(10, 10))\n", "with fits.open(os.path.join('../../', custom_run_image3, i2d_name)) as i2d:\n", - " ax.imshow(i2d[1].data, origin='lower', aspect='auto', vmin=0, vmax=np.nanmax(i2d[1].data)*0.01)\n", + " ax2.imshow(i2d[1].data, origin='lower', aspect='auto', vmin=0, vmax=np.nanmax(i2d[1].data)*0.01)\n", + " ax2.set_title(f\"{os.path.basename(i2d_name).split('_i2d')[0]}\")\n", "\n", "# also plot the associated catalog\n", "cat = Table.read(cat_name)\n", @@ -1014,24 +1069,25 @@ "\n", "for color, sources in zip(['darkorange', 'black'], [extended_sources, point_sources]):\n", " # plotting the sources\n", - " ax.scatter(sources['xcentroid'], sources['ycentroid'], s=40, facecolors='None', edgecolors=color, alpha=0.9, lw=2)\n", + " ax2.scatter(sources['xcentroid'], sources['ycentroid'], s=40, facecolors='None', edgecolors=color, alpha=0.9, lw=2)\n", "\n", " # adding source labels \n", " for i, source_num in enumerate(sources['label']):\n", - " ax.annotate(source_num, \n", - " (sources['xcentroid'][i]+0.5, sources['ycentroid'][i]+0.5), \n", - " fontsize=14,\n", - " color=color)\n", - " " + " ax2.annotate(source_num, \n", + " (sources['xcentroid'][i]+0.5, sources['ycentroid'][i]+0.5), \n", + " fontsize=14,\n", + " color=color)\n", + "\n", + "fig.tight_layout()" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "b196ecdb-4a07-4534-8f15-7dc0f27824af", + "cell_type": "markdown", + "id": "d252a65a-1be8-4e0b-ae67-80cac8fc4770", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "\"Space" + ] } ], "metadata": { From e26e10abea050aca781e44d824066c4db06269c0 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 2 Feb 2024 12:18:01 -0500 Subject: [PATCH 27/62] minor style updates to 00 and 01 --- .../00_niriss_mast_query_data_setup.ipynb | 2 +- .../01_niriss_wfss_image2_image3.ipynb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index 4e8bc007d..95007b16d 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -5,7 +5,7 @@ "id": "592da8da", "metadata": {}, "source": [ - "# Get observations from program ID\n", + "# Download WFSS Data\n", "This notebook uses the python [astroquery.mast Observations](https://astroquery.readthedocs.io/en/latest/mast/mast_obsquery.html) class of the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three [NIRISS filters](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrumentation/niriss-pupil-and-filter-wheels): F115W, F150W, and F200W using both GR150R and GR150C grisms.\n", "\n", "**Use case**: use MAST to download data products.
\n", diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 3e1063a6f..9b206b782 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -991,6 +991,14 @@ "source": [ "Once we have an updated source catalog that we are content with, we can move on to the spec2 step of the pipeline. It likely will be necessary to come back to this step after running spec2." ] + }, + { + "cell_type": "markdown", + "id": "d73170cc-0e25-44d2-ac6b-0ba56e8122a4", + "metadata": {}, + "source": [ + "\"Space" + ] } ], "metadata": { From 23493aa0ac510a6696c803b9236112eaeacc8cdc Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 2 Feb 2024 12:20:10 -0500 Subject: [PATCH 28/62] Removing extra notebooks for now --- .../07_visualize_level2_mosviz.ipynb | 618 ---------- .../08_specviz2d_extract1d.ipynb | 386 ------ .../09_compare_extractions.ipynb | 1088 ----------------- 3 files changed, 2092 deletions(-) delete mode 100644 notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb delete mode 100644 notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb delete mode 100755 notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb diff --git a/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb b/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb deleted file mode 100644 index 103261593..000000000 --- a/notebooks/NIRISS_WFSS_advanced/07_visualize_level2_mosviz.ipynb +++ /dev/null @@ -1,618 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "a7b71e63", - "metadata": {}, - "source": [ - "# Visualize level2 output in Mosviz\n", - "I will use a different instance of Mosviz per filter.\n", - "\n", - "**Author note**:\n", - "- worth including GR150C too?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fab6cc54", - "metadata": {}, - "outputs": [], - "source": [ - "# utility\n", - "import os\n", - "from astropy.table import Table, join\n", - "import numpy as np\n", - "import glob\n", - "import shutil\n", - "\n", - "# jdaviz\n", - "from jdaviz import Mosviz" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7a24e3f9", - "metadata": {}, - "outputs": [], - "source": [ - "# Update appropriately\n", - "data_dir_images = \"/Users/cpacifici/DATA/NGDEEP/image3/\"\n", - "data_dir_out_spec2 = \"/Users/cpacifici/DATA/NGDEEP/spec2/\"" - ] - }, - { - "cell_type": "markdown", - "id": "067cf0c4", - "metadata": {}, - "source": [ - "## F115W GR150R" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b00e9d9d", - "metadata": {}, - "outputs": [], - "source": [ - "directimage = data_dir_images + 'jw02079-o004_t001_niriss_clear-f115w_i2d.fits'\n", - "catalogfile = data_dir_images + 'jw02079-o004_t001_niriss_clear-f115w_cat.ecsv'\n", - "calfile = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_cal.fits'\n", - "x1dfile = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_x1d.fits'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b016350e", - "metadata": {}, - "outputs": [], - "source": [ - "dir_mosviz = './mosviz_stuff'\n", - "if os.path.exists(dir_mosviz):\n", - " shutil.rmtree(dir_mosviz)\n", - "\n", - "os.mkdir(dir_mosviz)\n", - "\n", - "shutil.copy(directimage, dir_mosviz)\n", - "shutil.copy(catalogfile, dir_mosviz)\n", - "shutil.copy(calfile, dir_mosviz)\n", - "shutil.copy(x1dfile, dir_mosviz)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0b7f1cc0", - "metadata": {}, - "outputs": [], - "source": [ - "mosviz1 = Mosviz()\n", - "mosviz1.load_data(directory=dir_mosviz, instrument='niriss')\n", - "mosviz1.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "128aec4b", - "metadata": {}, - "outputs": [], - "source": [ - "plotopt = mosviz1.plugins['Plot Options']\n", - "plotopt.viewer = 'image-viewer'\n", - "plotopt.stretch_vmin = 0.15\n", - "plotopt.stretch_vmax = 0.4\n", - "\n", - "plotopt.viewer = 'spectrum-2d-viewer'\n", - "plotopt.stretch_vmin = 0\n", - "plotopt.stretch_vmax = 2" - ] - }, - { - "cell_type": "markdown", - "id": "3a13745c", - "metadata": {}, - "source": [ - "We add a column to the table to take notes and mark the interesting objects with emission lines." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31703246", - "metadata": {}, - "outputs": [], - "source": [ - "mosviz1.add_column('Notes')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eed32ee9", - "metadata": {}, - "outputs": [], - "source": [ - "# Filter F115W grism GR150R\n", - "mosviz1.update_column('Notes', 'line', row=23) # ID 216\n", - "mosviz1.update_column('Notes', 'line maybe', row=25) # ID 243\n", - "mosviz1.update_column('Notes', 'line maybe', row=27) # ID 188\n", - "mosviz1.update_column('Notes', 'line', row=35) # ID 250\n", - "mosviz1.update_column('Notes', 'line', row=53) # ID 21\n", - "mosviz1.update_column('Notes', 'line', row=56) # ID 100\n", - "mosviz1.update_column('Notes', 'line', row=77) # ID 23\n", - "mosviz1.update_column('Notes', 'line', row=85) # ID 71\n", - "mosviz1.update_column('Notes', 'line', row=86) # ID 179\n", - "mosviz1.update_column('Notes', 'line', row=99) # ID 245\n", - "mosviz1.update_column('Notes', 'line maybe', row=100) # ID 109\n", - "mosviz1.update_column('Notes', 'line', row=117) # ID 22\n", - "mosviz1.update_column('Notes', 'line', row=134) # ID 2\n", - "mosviz1.update_column('Notes', 'line', row=137) # ID 4\n", - "mosviz1.update_column('Notes', 'line maybe', row=157) # ID 113\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a3807fd8", - "metadata": {}, - "outputs": [], - "source": [ - "table_f115w_grr = mosviz1.to_table()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "49115f40", - "metadata": {}, - "outputs": [], - "source": [ - "# Save table to file\n", - "table_f115w_grr.write('./f115w_gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" - ] - }, - { - "cell_type": "markdown", - "id": "765e6081", - "metadata": {}, - "source": [ - "## F150W GR150R" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6cfbcda7", - "metadata": {}, - "outputs": [], - "source": [ - "directimage = data_dir_images + 'jw02079-o004_t001_niriss_clear-f150w_i2d.fits'\n", - "catalogfile = data_dir_images + 'jw02079-o004_t001_niriss_clear-f150w_cat.ecsv'\n", - "calfile = data_dir_out_spec2 + 'jw02079004002_09101_00001_nis_cal.fits'\n", - "x1dfile = data_dir_out_spec2 + 'jw02079004002_09101_00001_nis_x1d.fits'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d920b143", - "metadata": {}, - "outputs": [], - "source": [ - "dir_mosviz = './mosviz_stuff'\n", - "if os.path.exists(dir_mosviz):\n", - " shutil.rmtree(dir_mosviz)\n", - "\n", - "os.mkdir(dir_mosviz)\n", - "\n", - "shutil.copy(directimage, dir_mosviz)\n", - "shutil.copy(catalogfile, dir_mosviz)\n", - "shutil.copy(calfile, dir_mosviz)\n", - "shutil.copy(x1dfile, dir_mosviz)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9f6ca5b1", - "metadata": {}, - "outputs": [], - "source": [ - "mosviz2 = Mosviz()\n", - "mosviz2.load_data(directory=dir_mosviz, instrument='niriss')\n", - "mosviz2.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99fb216e", - "metadata": {}, - "outputs": [], - "source": [ - "plotopt = mosviz2.plugins['Plot Options']\n", - "plotopt.viewer = 'image-viewer'\n", - "plotopt.stretch_vmin = 0.15\n", - "plotopt.stretch_vmax = 0.5\n", - "\n", - "plotopt.viewer = 'spectrum-2d-viewer'\n", - "plotopt.stretch_vmin = 0\n", - "plotopt.stretch_vmax = 3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b3aac083", - "metadata": {}, - "outputs": [], - "source": [ - "mosviz2.add_column('Notes')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5409284c", - "metadata": {}, - "outputs": [], - "source": [ - "# Filter F150W grism GR150R\n", - "mosviz2.update_column('Notes', 'line', row=14) # ID 170\n", - "mosviz2.update_column('Notes', 'line', row=21) # ID 150\n", - "mosviz2.update_column('Notes', 'maybe line', row=34) # ID 38\n", - "mosviz2.update_column('Notes', 'line', row=36) # ID 195\n", - "mosviz2.update_column('Notes', 'maybe line', row=44) # ID 184\n", - "mosviz2.update_column('Notes', 'maybe line', row=47) # ID 213\n", - "mosviz2.update_column('Notes', 'maybe line', row=57) # ID 25\n", - "mosviz2.update_column('Notes', 'line', row=60) # ID 218\n", - "mosviz2.update_column('Notes', 'line', row=62) # ID 21\n", - "mosviz2.update_column('Notes', 'line', row=63) # ID 37\n", - "mosviz2.update_column('Notes', 'maybe line', row=66) # ID 145\n", - "mosviz2.update_column('Notes', 'maybe line', row=70) # ID 19\n", - "mosviz2.update_column('Notes', 'line', row=71) # ID 207\n", - "mosviz2.update_column('Notes', 'line', row=73) # ID 15\n", - "mosviz2.update_column('Notes', 'line', row=74) # ID 63\n", - "mosviz2.update_column('Notes', 'line', row=77) # ID 109 # promising!\n", - "mosviz2.update_column('Notes', 'maybe line', row=81) # ID 23\n", - "mosviz2.update_column('Notes', 'maybe line', row=83) # ID 54\n", - "mosviz2.update_column('Notes', 'maybe line', row=84) # ID 190\n", - "mosviz2.update_column('Notes', 'maybe line', row=100) # ID 114\n", - "mosviz2.update_column('Notes', 'line', row=106) # ID 193 \n", - "mosviz2.update_column('Notes', 'line', row=110) # ID 135 \n", - "mosviz2.update_column('Notes', 'line', row=128) # ID 132 \n", - "mosviz2.update_column('Notes', 'maybe line', row=130) # ID 149\n", - "mosviz2.update_column('Notes', 'line', row=138) # ID 7 \n", - "mosviz2.update_column('Notes', 'line', row=139) # ID 44 \n", - "mosviz2.update_column('Notes', 'line', row=142) # ID 2 \n", - "mosviz2.update_column('Notes', 'maybe line', row=157) # ID 16\n", - "mosviz2.update_column('Notes', 'line', row=159) # ID 28 \n", - "mosviz2.update_column('Notes', 'line', row=161) # ID 41 \n", - "mosviz2.update_column('Notes', 'line', row=162) # ID 93 \n", - "mosviz2.update_column('Notes', 'maybe line', row=176) # ID 11\n", - "mosviz2.update_column('Notes', 'maybe line', row=177) # ID 113\n", - "mosviz2.update_column('Notes', 'line', row=180) # ID 27 \n", - "mosviz2.update_column('Notes', 'line', row=194) # ID 86 \n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d03b024a", - "metadata": {}, - "outputs": [], - "source": [ - "table_f150w_grr = mosviz2.to_table()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2acaebe5", - "metadata": {}, - "outputs": [], - "source": [ - "# Save table to file\n", - "table_f150w_grr.write('./f150w_gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" - ] - }, - { - "cell_type": "markdown", - "id": "0fd0bc12", - "metadata": {}, - "source": [ - "## F200W GR150R" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce46b974", - "metadata": {}, - "outputs": [], - "source": [ - "directimage = data_dir_images + 'jw02079-o004_t001_niriss_clear-f200w_i2d.fits'\n", - "catalogfile = data_dir_images + 'jw02079-o004_t001_niriss_clear-f200w_cat.ecsv'\n", - "calfile = data_dir_out_spec2 + 'jw02079004003_03101_00001_nis_cal.fits'\n", - "x1dfile = data_dir_out_spec2 + 'jw02079004003_03101_00001_nis_x1d.fits'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "849aad09", - "metadata": {}, - "outputs": [], - "source": [ - "dir_mosviz = './mosviz_stuff'\n", - "if os.path.exists(dir_mosviz):\n", - " shutil.rmtree(dir_mosviz)\n", - "\n", - "os.mkdir(dir_mosviz)\n", - "\n", - "shutil.copy(directimage, dir_mosviz)\n", - "shutil.copy(catalogfile, dir_mosviz)\n", - "shutil.copy(calfile, dir_mosviz)\n", - "shutil.copy(x1dfile, dir_mosviz)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1716821d", - "metadata": {}, - "outputs": [], - "source": [ - "mosviz3 = Mosviz()\n", - "mosviz3.load_data(directory=dir_mosviz, instrument='niriss')\n", - "mosviz3.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4314b1f4", - "metadata": {}, - "outputs": [], - "source": [ - "plotopt = mosviz3.plugins['Plot Options']\n", - "plotopt.viewer = 'image-viewer'\n", - "plotopt.stretch_vmin = 0.15\n", - "plotopt.stretch_vmax = 0.5\n", - "\n", - "plotopt.viewer = 'spectrum-2d-viewer'\n", - "plotopt.stretch_vmin = 0\n", - "plotopt.stretch_vmax = 4" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b086e2ce", - "metadata": {}, - "outputs": [], - "source": [ - "mosviz3.add_column('Notes')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "821fd610", - "metadata": {}, - "outputs": [], - "source": [ - "# Filter F200W grism GR150R\n", - "mosviz3.update_column('Notes', 'maybe line', row=25) # ID 216\n", - "mosviz3.update_column('Notes', 'line', row=39) # ID 123 # base key\n", - "mosviz3.update_column('Notes', 'line', row=48) # ID 92 \n", - "mosviz3.update_column('Notes', 'line', row=53) # ID 146 \n", - "mosviz3.update_column('Notes', 'maybe line', row=57) # ID 116\n", - "mosviz3.update_column('Notes', 'line', row=65) # ID 37\n", - "mosviz3.update_column('Notes', 'line', row=73) # ID 155\n", - "mosviz3.update_column('Notes', 'line', row=81) # ID 192\n", - "mosviz3.update_column('Notes', 'line', row=97) # ID 109\n", - "mosviz3.update_column('Notes', 'line', row=102) # ID 201\n", - "mosviz3.update_column('Notes', 'line', row=107) # ID 149\n", - "mosviz3.update_column('Notes', 'maybe line', row=114) # ID 98\n", - "mosviz3.update_column('Notes', 'line', row=116) # ID 52\n", - "mosviz3.update_column('Notes', 'maybe line', row=121) # ID 169\n", - "mosviz3.update_column('Notes', 'line', row=129) # ID 32\n", - "mosviz3.update_column('Notes', 'line', row=135) # ID 16\n", - "mosviz3.update_column('Notes', 'maybe line', row=139) # ID 136\n", - "mosviz3.update_column('Notes', 'line', row=165) # ID 120\n", - "mosviz3.update_column('Notes', 'line', row=173) # ID 41\n", - "mosviz3.update_column('Notes', 'line', row=188) # ID 93\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ae6b5fa0", - "metadata": {}, - "outputs": [], - "source": [ - "table_f200w_grr = mosviz3.to_table()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c47b96a7", - "metadata": {}, - "outputs": [], - "source": [ - "# Save table to file\n", - "table_f200w_grr.write('./f200w_gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" - ] - }, - { - "cell_type": "markdown", - "id": "f9fe8290", - "metadata": {}, - "source": [ - "### Delete files not needed anymore" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5d68b90f", - "metadata": {}, - "outputs": [], - "source": [ - "if os.path.exists(dir_mosviz):\n", - " shutil.rmtree(dir_mosviz)" - ] - }, - { - "cell_type": "markdown", - "id": "6e42598e", - "metadata": {}, - "source": [ - "## Combine catalogs and flags" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eafd7e06", - "metadata": {}, - "outputs": [], - "source": [ - "table_f115w_grr.sort(['Identifier'])\n", - "table_f150w_grr.sort(['Identifier'])\n", - "table_f200w_grr.sort(['Identifier'])" - ] - }, - { - "cell_type": "markdown", - "id": "c2d812b3", - "metadata": {}, - "source": [ - "### Remove non-useful columns" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "68576c23", - "metadata": {}, - "outputs": [], - "source": [ - "table_f115w_grr.remove_columns(['Table Index', '1D Spectra', '2D Spectra', 'Images', 'Filter/Grating', 'Redshift'])\n", - "table_f150w_grr.remove_columns(['Table Index', '1D Spectra', '2D Spectra', 'Images', 'Filter/Grating', 'Redshift'])\n", - "table_f200w_grr.remove_columns(['Table Index', '1D Spectra', '2D Spectra', 'Images', 'Filter/Grating', 'Redshift'])" - ] - }, - { - "cell_type": "markdown", - "id": "37ebf01c", - "metadata": {}, - "source": [ - "### Rename notes column" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dd14d9ba", - "metadata": {}, - "outputs": [], - "source": [ - "table_f115w_grr.rename_column('Notes', 'Notes_F115W')\n", - "table_f150w_grr.rename_column('Notes', 'Notes_F150W')\n", - "table_f200w_grr.rename_column('Notes', 'Notes_F200W')" - ] - }, - { - "cell_type": "markdown", - "id": "d43b7202", - "metadata": {}, - "source": [ - "### Join tables" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f3e2c095", - "metadata": {}, - "outputs": [], - "source": [ - "table_flags = join(table_f115w_grr, table_f150w_grr, keys='Identifier')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce4f93b5", - "metadata": {}, - "outputs": [], - "source": [ - "table_flags = join(table_flags, table_f200w_grr, keys='Identifier')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "65c4f85e", - "metadata": {}, - "outputs": [], - "source": [ - "table_flags" - ] - }, - { - "cell_type": "markdown", - "id": "47cb9a52", - "metadata": {}, - "source": [ - "### Write out flag file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3ff34311", - "metadata": {}, - "outputs": [], - "source": [ - "table_flags.write('./gr150r_objects_flags.ecsv', format='ascii.ecsv', overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c02ecbd0", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb b/notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb deleted file mode 100644 index 0deef5547..000000000 --- a/notebooks/NIRISS_WFSS_advanced/08_specviz2d_extract1d.ipynb +++ /dev/null @@ -1,386 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "08d94d39", - "metadata": {}, - "source": [ - "# Spectral extraction" - ] - }, - { - "cell_type": "markdown", - "id": "e88b5d78", - "metadata": {}, - "source": [ - "- select the object(s) with lines in all grisms\n", - "- extract the three spectra with specreduce\n", - "\n", - "**Author notes**:\n", - "- Combine all the exposures for a single target\n", - "- Find something more automatic on multiple spectra using specreduce?\n", - "- Extract with boxcar and optimal (also self profile)\n", - "- Still need to take care of NaNs in specreduce\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e3b32f0d", - "metadata": {}, - "outputs": [], - "source": [ - "# utility\n", - "import os\n", - "import glob\n", - "import shutil\n", - "import numpy as np\n", - "\n", - "# astropy\n", - "from astropy.table import Table, join\n", - "from astropy.io import fits\n", - "from astropy import units as u\n", - "\n", - "# jdaviz\n", - "from jdaviz import Specviz2d, Specviz\n", - "\n", - "# specutils\n", - "from specutils import Spectrum1D\n", - "\n", - "# \n", - "from matplotlib import pyplot as plt\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d04f1bb", - "metadata": {}, - "outputs": [], - "source": [ - "# Update appropriately\n", - "data_dir_images = \"/Users/cpacifici/DATA/NGDEEP/image3/\"\n", - "data_dir_out_spec2 = \"/Users/cpacifici/DATA/NGDEEP/spec2/\"" - ] - }, - { - "cell_type": "markdown", - "id": "833b183f", - "metadata": {}, - "source": [ - "## Select files and interesting object" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5b199602", - "metadata": {}, - "outputs": [], - "source": [ - "calfile_f115w = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_cal.fits'\n", - "calfile_f150w = data_dir_out_spec2 + 'jw02079004002_09101_00001_nis_cal.fits'\n", - "calfile_f200w = data_dir_out_spec2 + 'jw02079004003_03101_00001_nis_cal.fits'\n", - "calfiles = [calfile_f115w, calfile_f150w, calfile_f200w]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "660447e0", - "metadata": {}, - "outputs": [], - "source": [ - "# My favorite object from the exploration in notebook 07\n", - "object_id = 109" - ] - }, - { - "cell_type": "markdown", - "id": "7caba000", - "metadata": {}, - "source": [ - "### Find correct extension in the files" - ] - }, - { - "cell_type": "markdown", - "id": "ec7b8cac", - "metadata": {}, - "source": [ - "**Developer note:** there might be a way to know how many SCI extensions there are in a fits file, but I do not know how. For now I hard code `201` because this is the number of objects in the catalog that was used to generate the `cal` files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2e6895ee", - "metadata": {}, - "outputs": [], - "source": [ - "ext_num = []\n", - "for file in calfiles:\n", - " with fits.open(file) as hdu:\n", - " print(file)\n", - " for ii in range(1, 202):\n", - " if hdu['SCI',ii].header['SOURCEID']==object_id:\n", - " ext_num.append(ii)\n", - "ext_num" - ] - }, - { - "cell_type": "markdown", - "id": "8daf2d8f", - "metadata": {}, - "source": [ - "## Load in Specviz2d and extract 1D spectrum\n", - "\n", - "**Developer note**: the NaNs are not yet treated properly by Specviz2d (and specreduce). Need to mask them to zeros before proceeding." - ] - }, - { - "cell_type": "markdown", - "id": "9273c73b", - "metadata": {}, - "source": [ - "### F115W" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0c637d66", - "metadata": {}, - "outputs": [], - "source": [ - "filter_num = 0\n", - "calimage = fits.open(calfiles[filter_num])\n", - "data = calimage['SCI',ext_num[filter_num]].data\n", - "data = np.transpose(data)\n", - "data = np.nan_to_num(data)\n", - "wave = calimage['WAVELENGTH',ext_num[filter_num]].data[:,0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8b7c0590", - "metadata": {}, - "outputs": [], - "source": [ - "spec2d = Spectrum1D(flux=data*u.Jy, spectral_axis=wave*u.um)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6b91e580", - "metadata": {}, - "outputs": [], - "source": [ - "specviz2d = Specviz2d()\n", - "specviz2d.load_data(spec2d)\n", - "specviz2d.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9f01a54f", - "metadata": {}, - "outputs": [], - "source": [ - "spec1d_f115w = specviz2d.get_data('Spectrum 1D')" - ] - }, - { - "cell_type": "markdown", - "id": "03aa7058", - "metadata": {}, - "source": [ - "### F150W" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a20e31d1", - "metadata": {}, - "outputs": [], - "source": [ - "filter_num = 1\n", - "calimage = fits.open(calfiles[filter_num])\n", - "data = calimage['SCI',ext_num[filter_num]].data\n", - "data = np.transpose(data)\n", - "data = np.nan_to_num(data)\n", - "wave = calimage['WAVELENGTH',ext_num[filter_num]].data[:,0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d3919e94", - "metadata": {}, - "outputs": [], - "source": [ - "spec2d = Spectrum1D(flux=data*u.Jy, spectral_axis=wave*u.um)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1ee3d806", - "metadata": {}, - "outputs": [], - "source": [ - "specviz2d2 = Specviz2d()\n", - "specviz2d2.load_data(spec2d)\n", - "specviz2d2.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e161b0e", - "metadata": {}, - "outputs": [], - "source": [ - "spec1d_f150w = specviz2d2.get_data('Spectrum 1D')" - ] - }, - { - "cell_type": "markdown", - "id": "71a9814f", - "metadata": {}, - "source": [ - "### F200W" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ea2e0c45", - "metadata": {}, - "outputs": [], - "source": [ - "filter_num = 2\n", - "calimage = fits.open(calfiles[filter_num])\n", - "data = calimage['SCI',ext_num[filter_num]].data\n", - "data = np.transpose(data)\n", - "data = np.nan_to_num(data)\n", - "wave = calimage['WAVELENGTH',ext_num[filter_num]].data[:,0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de1815d0", - "metadata": {}, - "outputs": [], - "source": [ - "spec2d = Spectrum1D(flux=data*u.Jy, spectral_axis=wave*u.um)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3b2a6e89", - "metadata": {}, - "outputs": [], - "source": [ - "specviz2d3 = Specviz2d()\n", - "specviz2d3.load_data(spec2d)\n", - "specviz2d3.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "613b1402", - "metadata": {}, - "outputs": [], - "source": [ - "spec1d_f200w = specviz2d3.get_data('Spectrum 1D')" - ] - }, - { - "cell_type": "markdown", - "id": "5ce78c0f", - "metadata": {}, - "source": [ - "## Open in Specviz" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a1ab9cf", - "metadata": {}, - "outputs": [], - "source": [ - "specviz = Specviz()\n", - "specviz.load_data(spec1d_f115w, data_label='f115w')\n", - "specviz.load_data(spec1d_f150w, data_label='f150w')\n", - "specviz.load_data(spec1d_f200w, data_label='f200w')\n", - "specviz.show()" - ] - }, - { - "cell_type": "markdown", - "id": "e974d11f", - "metadata": {}, - "source": [ - "Load a line list and measure the redshift of the target (spoiler: z=2.23)" - ] - }, - { - "cell_type": "markdown", - "id": "684e262b", - "metadata": {}, - "source": [ - "### Save spectra as fits files" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "695c3d5c", - "metadata": {}, - "outputs": [], - "source": [ - "spec1d_f115w.write('source109_f115w.fits')\n", - "spec1d_f150w.write('source109_f150w.fits')\n", - "spec1d_f200w.write('source109_f200w.fits')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8b59d88b", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb b/notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb deleted file mode 100755 index f5afd17a5..000000000 --- a/notebooks/NIRISS_WFSS_advanced/09_compare_extractions.ipynb +++ /dev/null @@ -1,1088 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Compare custom extraction and pipeline extraction\n", - "\n", - "**Author note**:\n", - "- Need to run spec3?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# utility\n", - "import os\n", - "import glob\n", - "import shutil\n", - "import numpy as np\n", - "\n", - "# astropy\n", - "from astropy.table import Table, join\n", - "from astropy.io import fits\n", - "from astropy import units as u\n", - "\n", - "# specutils\n", - "from specutils import Spectrum1D\n", - "\n", - "# matplotlib\n", - "from matplotlib import pyplot as plt\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Update appropriately\n", - "data_dir_out_spec2 = \"/Users/cpacifici/DATA/NGDEEP/spec2/\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x1dfile_f115w = data_dir_out_spec2 + 'jw02079004001_03101_00001_nis_x1d.fits'\n", - "x1dfile_f150w = data_dir_out_spec2 + 'jw02079004002_09101_00001_nis_x1d.fits'\n", - "x1dfile_f200w = data_dir_out_spec2 + 'jw02079004003_03101_00001_nis_x1d.fits'\n", - "x1dfiles = [x1dfile_f115w, x1dfile_f150w, x1dfile_f200w]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# My favorite object from the exploration in notebook 07\n", - "object_id = 109" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ext_num = []\n", - "spec1d = []\n", - "for file in x1dfiles:\n", - " with fits.open(file) as hdu:\n", - " print(file)\n", - " for ii in range(1, 202):\n", - " if hdu['EXTRACT1D',ii].header['SOURCEID']==object_id:\n", - " ext_num.append(ii)\n", - " spec1d.append(hdu['EXTRACT1D',ii].data)\n", - "ext_num" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "custom_ext_f115w = fits.open('source109_f115w.fits')\n", - "custom_ext_f150w = fits.open('source109_f150w.fits')\n", - "custom_ext_f200w = fits.open('source109_f200w.fits')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(16,10))\n", - "plt.plot(spec1d[0]['WAVELENGTH'], spec1d[0]['FLUX'], '-', color='blue')\n", - "plt.plot(custom_ext_f115w[1].data['wavelength'], custom_ext_f115w[1].data['flux']/10000000, '--', color='blue')\n", - "plt.plot(spec1d[1]['WAVELENGTH'], spec1d[1]['FLUX'], '-', color='green')\n", - "plt.plot(custom_ext_f150w[1].data['wavelength'], custom_ext_f150w[1].data['flux']/10000000, '--', color='green')\n", - "plt.plot(spec1d[2]['WAVELENGTH'], spec1d[2]['FLUX'], '-', color='red')\n", - "plt.plot(custom_ext_f200w[1].data['wavelength'], custom_ext_f200w[1].data['flux']/10000000, '--', color='red')\n", - "plt.ylim(0,0.00002)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Not finished...still working on the rest" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# WFSS Spectra Part 0: Optimal Extraction" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Use case:** optimal extraction of grism spectra; redshift measurement; emission-line maps. Simplified version of [JDox Science Use Case # 33](https://jwst-docs.stsci.edu/near-infrared-imager-and-slitless-spectrograph/niriss-example-science-programs/niriss-wfss-with-nircam-parallel-imaging-of-galaxies-in-lensing-clusters).
\n", - "**Data:** JWST simulated NIRISS images from [MIRAGE](https://jwst-docs.stsci.edu/jwst-other-tools/mirage-data-simulator), run through the [JWST calibration pipeline](https://jwst-pipeline.readthedocs.io/en/latest/); galaxy cluster.
\n", - "**Tools:** specutils, astropy, pandas, emcee, lmfit, corner, h5py.
\n", - "**Cross-intrument:** NIRSpec
\n", - "**Documentation:** This notebook is part of a STScI's larger [post-pipeline Data Analysis Tools Ecosystem](https://jwst-docs.stsci.edu/jwst-post-pipeline-data-analysis).
\n", - "\n", - "\n", - "## Introduction\n", - "\n", - "This notebook is 1 of 4 in a set focusing on NIRISS WFSS data:\n", - " 1. 1D optimal extraction since the JWST pipeline only provides a box extraction. Optimal extraction improves S/N of spectra for faint sources.\n", - " 2. Combine and normalize 1D spectra.\n", - " 3. Cross correlate galaxy with template to get redshift.\n", - " 4. Spatially resolved emission line map.\n", - "\n", - "This notebook will start with [post-pipeline](https://jwst-docs.stsci.edu/jwst-data-reduction-pipeline) products of NIRISS WFSS, 2D rectified spectra, from spec level3.\n", - "\n", - "Optimal extraction requires source morphology along the cross-dispersion direction, which will be retrieved from direct images taken along with WFSS observations. Morphology along dispersion direction is also essential to infer the spectral resolution, which will be obtained using template fitting to get redshift and stellar population in notebook #3 of this set.\n", - "\n", - "**Note:** We here assume reduction of the 2D rectified spectrum has been performed at a decent level, i.e. there is no contaminating flux from other sources on the target 2D spectrum, and that background is already subtracted." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import numpy as np\n", - "from scipy.ndimage import rotate\n", - "from scipy.optimize import curve_fit\n", - "\n", - "\n", - "from astropy.convolution import Gaussian2DKernel\n", - "from astropy.io import fits\n", - "from astropy.stats import gaussian_fwhm_to_sigma\n", - "from astropy.table import QTable\n", - "import astropy.units as u\n", - "from astropy.visualization import make_lupton_rgb, SqrtStretch, ImageNormalize, simple_norm\n", - "import astropy.wcs as wcs\n", - "from astropy.io import ascii\n", - "\n", - "from specutils import Spectrum1D\n", - "from astropy.nddata import StdDevUncertainty\n", - "\n", - "import specutils\n", - "print('specutils', specutils.__version__)\n", - "\n", - "import astropy\n", - "print('astropy', astropy.__version__)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### The version should be \n", - "- specutils 1.0\n", - "- astropy 4.0.1.post1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import matplotlib as mpl\n", - "\n", - "mpl.rcParams['savefig.dpi'] = 80\n", - "mpl.rcParams['figure.dpi'] = 80" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 0.Download and load data:\n", - "These include pipeline processed data for NIRISS, as well as photometric catalog from image step3." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not os.path.exists('./pipeline_products'):\n", - " import zipfile\n", - " import urllib.request\n", - " boxlink = 'https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/NIRISS_lensing_cluster/pipeline_products.zip'\n", - " boxfile = './pipeline_products.zip'\n", - " urllib.request.urlretrieve(boxlink, boxfile)\n", - " zf = zipfile.ZipFile(boxfile, 'r')\n", - " zf.extractall()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "DIR_DATA = './pipeline_products/'\n", - "\n", - "# Output directory;\n", - "DIR_OUT = './output/'\n", - "if not os.path.exists(DIR_OUT):\n", - " os.mkdir(DIR_OUT)\n", - "\n", - "# Filter for detection and science image;\n", - "filt_det = 'f200w'\n", - "\n", - "# Image array from direct image. This is for optimal extraction and masking.\n", - "# This image should already be sky-subtracted; otherwise, you will encounter a wrong result with optimal extraction.\n", - "infile = '%sl3_nis_%s_i2d_skysub.fits'%(DIR_DATA,filt_det)\n", - "hdu = fits.open(infile)\n", - "\n", - "# This is just for error array;\n", - "infile = '%sl3_nis_%s_i2d.fits'%(DIR_DATA,filt_det)\n", - "hdu_err = fits.open(infile)\n", - "\n", - "data = hdu[0].data\n", - "imwcs = wcs.WCS(hdu[0].header, hdu)\n", - "\n", - "err = hdu_err[2].data\n", - "weight = 1/np.square(err)\n", - "\n", - "# Segmentation map;\n", - "# This can be prepared by running Photutils, if the pipeline does not generate one.\n", - "segfile = '%sl3_nis_%s_i2d_seg.fits'%(DIR_DATA, filt_det)\n", - "seghdu = fits.open(segfile)\n", - "segdata = seghdu[0].data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load catalog from image level3;\n", - "# to obtain source position in pixel coordinate.\n", - "catfile = '%sl3_nis_%s_cat.ecsv'%(DIR_DATA, filt_det)\n", - "fd = ascii.read('%s'%catfile)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Make a broadband flux catalog.\n", - "- For a convenient reason, here we compile catalogs into a flux catalog, which will be used in the following notebook (01b).\n", - "- To run this cell, you will need a photometric catalog of sources, that list sources position and flux for each filter. For now, I use this catalog prepared in another notebook. (\"sources_extend_01.cat\")\n", - "- This catalog can also be used for generic phot-z/SED fitting softwares, like EAZY and gsf (see notebook No.04).\n", - "\n", - "#### For now, we use an input catalog." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "filts = ['f115w', 'f150w', 'f200w', 'f090w', 'f435w', 'f606w', 'f814w', 'f105w', 'f125w', 'f140w', 'f160w']\n", - "eazy_filts = [309, 310, 311, 308, 1, 4, 6, 202, 203, 204, 205]\n", - "magzp = 25.0 # magnitude zeropoint in the catalog.\n", - "\n", - "# Read catalog;\n", - "fd_input = ascii.read('%ssources_extend_01.cat'%(DIR_DATA))\n", - "ra_input = fd_input['x_or_RA']\n", - "dec_input = fd_input['y_or_Dec']\n", - "\n", - "# Header;\n", - "fw = open('%sl3_nis_flux.cat'%(DIR_OUT), 'w')\n", - "fw.write('# id')\n", - "for ff in range(len(filts)):\n", - " fw.write(' F%d E%d'%(eazy_filts[ff], eazy_filts[ff]))\n", - "fw.write('\\n')\n", - "\n", - "# Contents;\n", - "for ii in range(len(fd['id'])):\n", - " \n", - " rtmp = np.sqrt((fd['sky_centroid'].ra.value[ii] - ra_input[:])**2 + (fd['sky_centroid'].dec.value[ii] - dec_input[:])**2)\n", - " iix = np.argmin(rtmp)\n", - " \n", - " for ff in range(len(filts)):\n", - " if ff == 0:\n", - " fw.write('%d'%(fd['id'][ii]))\n", - "\n", - " mag = fd_input['niriss_%s_magnitude'%filts[ff]][iix]\n", - " flux_nu = 10**((mag-magzp)/(-2.5))\n", - "\n", - " # Currently, the catalog does not provide proper error;\n", - " # Assuming 5% error for flux.\n", - " \n", - " flux_err_nu = flux_nu * 0.05\n", - "\n", - " fw.write(' %.5e %.5e'%(flux_nu, flux_err_nu))\n", - "\n", - " fw.write('\\n')\n", - "fw.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1.Load 2D spectrum;" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# Which filter, grating, and object?\n", - "filt = 'f200w'\n", - "\n", - "#grism = 'G150R'\n", - "grism = 'G150C'\n", - "\n", - "id = '00004'\n", - "\n", - "# Zero-indexed number for dither --- the test data here has two dither positions, so 0 or 1.\n", - "ndither = 0\n", - "\n", - "file_2d = '%sl3_nis_%s_%s_s%s_cal.fits'%(DIR_DATA, filt, grism, id)\n", - "hdu_2d = fits.open(file_2d)\n", - "\n", - "# Align grism direction\n", - "# - x-direction = Dispersion (wavelength) direction.\n", - "# - y-direction = Cross-dispersion.\n", - "# in this notebook.\n", - " \n", - "if grism == 'G150C':\n", - " # If spectrum is horizontal;\n", - " data_2d = hdu_2d[ndither*7+1].data\n", - " dq_2d = hdu_2d[ndither*7+2].data\n", - " err_2d = hdu_2d[ndither*7+3].data\n", - " wave_2d = hdu_2d[ndither*7+4].data\n", - "else:\n", - " data_2d = rotate(hdu_2d[ndither*7+1].data, 90)\n", - " dq_2d = rotate(hdu_2d[ndither*7+2].data, 90)\n", - " err_2d = rotate(hdu_2d[ndither*7+3].data, 90)\n", - " wave_2d = rotate(hdu_2d[ndither*7+4].data, 90)\n", - "\n", - "# Get position angle of observation;\n", - "hd_2d = hdu_2d[1].header\n", - "PA_V3 = hd_2d['PA_V3']\n", - "PA_V3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.imshow(data_2d, vmin=0, vmax=1000)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Get light profile of the source;\n", - "\n", - "# Again, y is for cross-dispersion, and x is for dispersion directions.\n", - "y2d,x2d = data_2d.shape[:]\n", - "\n", - "# Cut out segmentation map;\n", - "iix = np.where(fd['id']==int(id))[0][0]\n", - "\n", - "# Target position from image 3 catalog;\n", - "ycen = fd['ycentroid'][iix].value\n", - "xcen = fd['xcentroid'][iix].value\n", - "\n", - "# Cutout size = y direction of 2D spectrum;\n", - "rsq = y2d\n", - "\n", - "sci_cut = data[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", - "seg_cut = segdata[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", - "\n", - "# Rotate image for PA of Grism observation;\n", - "if grism == 'G150C':\n", - " sci_rot = rotate(sci_cut, PA_V3)\n", - "else:\n", - " sci_rot = rotate(sci_cut, PA_V3+90)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### WFSS grism is dispersed in a direction of x-axis in the plot below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.imshow(sci_rot, vmin=0, vmax=1.0)\n", - "plt.title('Direct image')\n", - "plt.xlabel('Wavelength direction >>>', color='r', fontsize=18)\n", - "plt.ylabel('Cross-dispersion direction >>>', color='r', fontsize=18)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2.Get light profile at different x position --- This will be used for optimal extraction." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for ii in range(sci_rot.shape[1]):\n", - " flux_tmp = sci_rot[:,ii]\n", - " xx_tmp = np.arange(0, len(sci_rot[:,ii]), 1)\n", - " plt.plot(xx_tmp, flux_tmp, label='x=%d'%ii)\n", - "plt.legend(loc=0, fontsize=8)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Sum along x (disperse) direction\n", - "flux_y = np.zeros(len(sci_rot[:,0]), 'float')\n", - "for ii in range(sci_rot.shape[0]):\n", - " flux_y[ii] = np.sum(sci_rot[ii,:])\n", - "\n", - "# Sky subtraction, if needed.\n", - "#sky = np.mean([flux_y[0], flux_y[-1]])\n", - "\n", - "# Normalize;\n", - "flux_y[:] /= flux_y.sum()\n", - "\n", - "plt.plot(xx_tmp, flux_y)\n", - "plt.xlabel('y-position')\n", - "plt.ylabel('Source Flux')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3.One-dimensional extraction;" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Show pipeline 1D extraction as an example;" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Normal extraction;\n", - "flux_disp1 = np.zeros(x2d, 'float')\n", - "err_disp1 = np.zeros(x2d, 'float')\n", - "wave_disp1 = np.zeros(x2d, 'float')\n", - " \n", - "for ii in range(x2d): # Wavelength direction.\n", - " mask_tmp = (dq_2d[:,ii] == 0) & (err_2d[:,ii]>0)\n", - "\n", - " # Sum within a box;\n", - " flux_disp1[ii] = np.sum(data_2d[:,ii][mask_tmp]) \n", - " err_disp1[ii] = np.sqrt(np.sum(err_2d[:,ii][mask_tmp]**2)) \n", - " wave_disp1[ii] = wave_2d[0,ii]\n", - "\n", - "plt.errorbar(wave_disp1, flux_disp1, yerr=err_disp1)\n", - "plt.xlim(1.7,2.3)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Optimal extraction;" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Following Horne(1986, PASP, 98, 609);\n", - "flux_disp = np.zeros(x2d, 'float')\n", - "err_disp = np.zeros(x2d, 'float')\n", - "wave_disp = np.zeros(x2d, 'float')\n", - "\n", - "# Sigma clipping.\n", - "sig = 5.0\n", - "\n", - "for ii in range(x2d): # ii : wavelength element.\n", - " # Mask; \n", - " # 1. DQ array\n", - " # 2. error value\n", - " # 3. CR detection\n", - " mask_tmp = (dq_2d[:,ii] == 0) & (err_2d[:,ii]>0) & ( (data_2d[:,ii] - flux_y[:] * flux_disp1[ii])**2 < sig**2 * err_2d[:,ii]**2)\n", - " ivar = 1. / err_2d[:,ii]**2\n", - "\n", - " num = flux_y[:] * data_2d[:,ii] * ivar\n", - " den = flux_y[:]**2 * ivar\n", - " flux_disp[ii] = num[mask_tmp].sum(axis=0) / den[mask_tmp].sum(axis=0)\n", - " err_disp[ii] = np.sqrt(1./den[mask_tmp].sum(axis=0))\n", - " wave_disp[ii] = wave_2d[0,ii]\n", - " \n", - "plt.errorbar(wave_disp, flux_disp, yerr=err_disp)\n", - "plt.xlim(1.7,2.3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Compare;\n", - "plt.errorbar(wave_disp, flux_disp, yerr=err_disp, color='r', label='Optimal')\n", - "plt.errorbar(wave_disp1, flux_disp1, yerr=err_disp1, color='b', alpha=0.5, label='Box')\n", - "plt.ylim(-10, 20000)\n", - "plt.legend(loc=0)\n", - "plt.xlabel('Wavelength')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4.Write 1d spectrum out to a file;" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_1d = '%sl3_nis_%s_%s_s%s_1d_opt.fits'%(DIR_OUT, filt, grism, id)\n", - "\n", - "# Now make it into a Spectrum1D instance.\n", - "obs = Spectrum1D(spectral_axis=wave_disp*u.um,\n", - " flux=flux_disp*u.MJy,\n", - " uncertainty=StdDevUncertainty(err_disp), unit='MJy')\n", - "obs.write(file_1d, format='tabular-fits', overwrite=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5.Light profile along x-axis = Resolution of dispersed spectrum;\n", - "As WFSS does not have a slit, any dispersed spectrum is affected by source morphology. The estimate on the effective spectral resolution will be needed in the following notebook. And we here try to estimate it beforehand;" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for ii in range(sci_rot.shape[0]):\n", - " flux_tmp = sci_rot[ii,:]\n", - " xx_tmp = np.arange(0, len(sci_rot[ii,:]), 1)\n", - " plt.plot(xx_tmp, flux_tmp, label='y=%d'%(ii))\n", - " \n", - "plt.legend(loc=1, fontsize=8)\n", - "plt.xlabel('Wavelength direction')\n", - "plt.title('Source light profile along dispersed direction', fontsize=14)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### *Unless you are interested in spatially resolved spectra, you can stack and get light profile as a good approximation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Sum along cross-disperse direction\n", - "flux_x = np.zeros(len(sci_rot[0,:]), 'float')\n", - "for ii in range(sci_rot.shape[0]):\n", - " flux_x[ii] = np.sum(sci_rot[:,ii])\n", - "\n", - "# Normalize;\n", - "flux_x[:] /= flux_x.sum()\n", - "\n", - "plt.plot(xx_tmp, flux_x, label='Convolution kernel')\n", - "plt.legend(loc=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Fit with a moffat function;" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Fitting function with Moffat\n", - "\n", - "# Moffat fnc.\n", - "def moffat(xx, A, x0, gamma, alp):\n", - " yy = A * (1. + (xx-x0)**2/gamma**2)**(-alp)\n", - " return yy\n", - "\n", - "def fit_mof(xx, lsf):\n", - " #xx = lsf * 0\n", - " #for ii in range(len(lsf)):\n", - " # xx[ii] = ii - len(lsf)/2.\n", - " popt, pcov = curve_fit(moffat, xx, lsf)\n", - " return popt\n", - "\n", - "def LSF_mof(xsf, lsf, f_plot=True):\n", - " '''\n", - " Input:\n", - " =======\n", - " xsf : x axis for the profile.\n", - " lsf : light profile. \n", - " '''\n", - " \n", - " #for ii in range(len(sci[0,:])):\n", - " # lsf[ii] = np.mean(sci_rot[int(height/2.-5):int(height/2.+5), ii])\n", - " # xsf[ii] = ii - len(lsf)/2.\n", - "\n", - " try:\n", - " A, xm, gamma, alpha = fit_mof(xsf, lsf)\n", - " except RuntimeError:\n", - " print('Fitting failed.')\n", - " A, xm, gamma, alpha = -1, -1, -1, -1\n", - " pass\n", - "\n", - " if A>0:\n", - " lsf_mod = moffat(xsf, A, 0, gamma, alpha)\n", - " \n", - " if f_plot:\n", - " yy = moffat(xsf, A, xm, gamma, alpha)\n", - " plt.plot(xsf, yy, 'r.', ls='-', label='Data')\n", - " plt.plot(xsf, lsf_mod, 'b+', ls='-', label='Model:$gamma=%.2f$\\n$alpha=%.2f$'%(gamma, alpha))\n", - " plt.legend()\n", - " plt.show()\n", - " \n", - " return A, xm, gamma, alpha" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# LSF, line spread function\n", - "iix_peak = np.argmax(flux_x)\n", - "xx_tmp_shift = xx_tmp - xx_tmp[iix_peak]\n", - "A, xm, gamma, alpha = LSF_mof(xx_tmp_shift, flux_x)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Write it down;\n", - "# Tha parameters are in unit of pixel.\n", - "fm = open('%sl3_nis_%s_%s_s%s_moffat.txt'%(DIR_OUT, filt, grism, id), 'w')\n", - "fm.write('# A x0 gamma alp\\n')\n", - "fm.write('# Moffat function\\n')\n", - "fm.write('%.3f %.3f %.3f %.3f\\n'%(A, xm, gamma, alpha))\n", - "\n", - "fm.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Repeat for other filters, other objects.\n", - "### The following big colum executes the same processes above for other filters and dither position." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "grism = 'G150C'\n", - "id = '00004'\n", - "DIR_OUT = './output/'\n", - "if not os.path.exists(DIR_OUT):\n", - " os.mkdir(DIR_OUT)\n", - "\n", - "filts = ['f115w', 'f150w', 'f200w']\n", - "ndithers = np.arange(0, 2, 1)\n", - "\n", - "sig = 5.0\n", - "\n", - "for filt in filts:\n", - " print(filt)\n", - " \n", - " # 2d spectrum;\n", - " file_2d = '%sl3_nis_%s_%s_s%s_cal.fits'%(DIR_DATA, filt, grism, id)\n", - " hdu_2d = fits.open(file_2d)\n", - "\n", - " for ndither in ndithers:\n", - " print(ndither)\n", - "\n", - " if grism == 'G150C':\n", - " # If spectrum is horizontal;\n", - " data_2d = hdu_2d[ndither*7+1].data\n", - " dq_2d = hdu_2d[ndither*7+2].data\n", - " err_2d = hdu_2d[ndither*7+3].data\n", - " wave_2d = hdu_2d[ndither*7+4].data\n", - " else:\n", - " data_2d = rotate(hdu_2d[ndither*7+1].data, 90)\n", - " dq_2d = rotate(hdu_2d[ndither*7+2].data, 90)\n", - " err_2d = rotate(hdu_2d[ndither*7+3].data, 90)\n", - " wave_2d = rotate(hdu_2d[ndither*7+4].data, 90)\n", - "\n", - " y2d,x2d = data_2d.shape[:]\n", - "\n", - " plt.close()\n", - " plt.imshow(data_2d, vmin=0, vmax=300)\n", - " plt.show()\n", - "\n", - " # Re-extract 2d image;\n", - " #if ndither == 0:\n", - " rsq = y2d\n", - " sci_cut = data[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", - " seg_cut = segdata[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", - "\n", - " # Not sure if the offset in extractioin box is bug ;\n", - " if grism == 'G150C':\n", - " sci_rot = rotate(sci_cut, PA_V3+0)\n", - " else:\n", - " sci_rot = rotate(sci_cut, PA_V3+0+90)\n", - "\n", - "\n", - " #\n", - " # This is for spectral resolution;\n", - " #\n", - " # Get light profile along the x-axis\n", - " for ii in range(sci_rot.shape[0]):\n", - " flux_tmp = sci_rot[ii,:]\n", - " xx_tmp = np.arange(0, len(sci_rot[ii,:]), 1)\n", - "\n", - " # Sum along cross-disperse direction\n", - " flux_x = np.zeros(len(sci_rot[0,:]), 'float')\n", - " for ii in range(sci_rot.shape[0]):\n", - " flux_x[ii] = np.sum(sci_rot[ii,:])\n", - "\n", - " # Normalize;\n", - " flux_x[:] /= flux_x.sum()\n", - "\n", - " # LSF\n", - " iix_peak = np.argmax(flux_x)\n", - " xx_tmp_shift = xx_tmp - xx_tmp[iix_peak]\n", - " A, xm, gamma, alpha = LSF_mof(xx_tmp_shift, flux_x)\n", - "\n", - " if ndither == 0:\n", - " # Write it down;\n", - " fm = open('%sl3_nis_%s_%s_s%s_moffat.txt'%(DIR_OUT, filt, grism, id), 'w')\n", - " fm.write('# A x0 gamma alp\\n')\n", - " fm.write('# Moffat function\\n')\n", - " fm.write('%.3f %.3f %.3f %.3f\\n'%(A, xm, gamma, alpha))\n", - " fm.close()\n", - "\n", - " #\n", - " # This is for Optimal extraction;\n", - " #\n", - " # Sum along x (disperse) direction\n", - " flux_y = np.zeros(len(sci_rot[:,0]), 'float')\n", - " for ii in range(sci_rot.shape[0]):\n", - " flux_y[ii] = np.sum(sci_rot[ii,:])\n", - " \n", - " # Normalize;\n", - " flux_y[:] /= flux_y.sum()\n", - "\n", - "\n", - " # Following Horne;\n", - " flux_disp = np.zeros(x2d, 'float')\n", - " err_disp = np.zeros(x2d, 'float')\n", - " wave_disp = np.zeros(x2d, 'float')\n", - "\n", - " for ii in range(x2d):\n", - " # Mask; \n", - " # 1. DQ array\n", - " # 2. error value\n", - " # 3. CR detection\n", - " mask_tmp = (dq_2d[:,ii] == 0) & (err_2d[:,ii] > 0)\n", - " ivar = 1. / err_2d[:,ii]**2\n", - "\n", - " num = flux_y[:] * data_2d[:,ii] * ivar \n", - " den = flux_y[:]**2 * ivar\n", - " flux_disp[ii] = num[mask_tmp].sum(axis=0)/den[mask_tmp].sum(axis=0)\n", - " err_disp[ii] = np.sqrt(1./den[mask_tmp].sum(axis=0))\n", - " wave_disp[ii] = wave_2d[0,ii]\n", - "\n", - "\n", - " plt.close()\n", - " con_plot = (wave_disp>0)\n", - " plt.errorbar(wave_disp[con_plot], flux_disp[con_plot], yerr=err_disp[con_plot])\n", - " plt.ylim(-0, 3000)\n", - " plt.show()\n", - "\n", - " # Wirte:\n", - " # Now make it into a Spectrum1D instance.\n", - " file_1d = '%sl3_nis_%s_%s_s%s_ndither%d_1d_opt.fits'%(DIR_OUT, filt, grism, id, ndither)\n", - "\n", - " if wave_disp[1] - wave_disp[0] < 0:\n", - " obs = Spectrum1D(spectral_axis=wave_disp[::-1]*u.um,\n", - " flux=flux_disp[::-1]*u.MJy,\n", - " uncertainty=StdDevUncertainty(err_disp[::-1]), unit='MJy')\n", - " else:\n", - " obs = Spectrum1D(spectral_axis=wave_disp*u.um,\n", - " flux=flux_disp*u.MJy,\n", - " uncertainty=StdDevUncertainty(err_disp), unit='MJy')\n", - " \n", - " obs.write(file_1d, format='tabular-fits', overwrite=True)\n", - " \n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Another object;\n", - "Absorption line galaxy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "grism = 'G150C'\n", - "id = '00003'\n", - "DIR_OUT = './output/'\n", - "\n", - "filts = ['f115w', 'f150w', 'f200w']\n", - "ndithers = np.arange(0,2,1) # There are four dithers in the data set;\n", - "\n", - "sig = 5.0\n", - "\n", - "for filt in filts:\n", - " print(filt)\n", - " # 2d spectrum;\n", - " file_2d = '%sl3_nis_%s_%s_s%s_cal.fits'%(DIR_DATA, filt, grism, id)\n", - " hdu_2d = fits.open(file_2d)\n", - "\n", - " for ndither in ndithers:\n", - " print(ndither)\n", - " if grism == 'G150C':\n", - " # If spectrum is horizontal;\n", - " data_2d = hdu_2d[ndither*7+1].data\n", - " dq_2d = hdu_2d[ndither*7+2].data\n", - " err_2d = hdu_2d[ndither*7+3].data\n", - " wave_2d = hdu_2d[ndither*7+4].data\n", - " else:\n", - " data_2d = rotate(hdu_2d[ndither*7+1].data, 90)\n", - " dq_2d = rotate(hdu_2d[ndither*7+2].data, 90)\n", - " err_2d = rotate(hdu_2d[ndither*7+3].data, 90)\n", - " wave_2d = rotate(hdu_2d[ndither*7+4].data, 90)\n", - "\n", - " y2d,x2d = data_2d.shape[:]\n", - "\n", - " plt.close()\n", - " plt.imshow(data_2d, vmin=0, vmax=20)\n", - " plt.show()\n", - "\n", - " # Re-extract 2d image;\n", - " #if ndither == 0:\n", - " rsq = y2d\n", - " sci_cut = data[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", - " seg_cut = segdata[int(ycen-rsq/2.+0.5):int(ycen+rsq/2.+0.5), int(xcen-rsq/2.+0.5):int(xcen+rsq/2.+0.5)]\n", - "\n", - " # Not sure if the offset in extractioin box is bug ;\n", - " if grism == 'G150C':\n", - " sci_rot = rotate(sci_cut, PA_V3+0)\n", - " else:\n", - " sci_rot = rotate(sci_cut, PA_V3+0+90)\n", - "\n", - " #\n", - " # This is for spectral resolution;\n", - " #\n", - " # Get light profile along the x-axis\n", - " for ii in range(sci_rot.shape[0]):\n", - " flux_tmp = sci_rot[ii,:]\n", - " xx_tmp = np.arange(0, len(sci_rot[ii,:]), 1)\n", - "\n", - " # Sum along cross-disperse direction\n", - " flux_x = np.zeros(len(sci_rot[0,:]), 'float')\n", - " for ii in range(sci_rot.shape[0]):\n", - " flux_x[ii] = np.sum(sci_rot[ii,:])\n", - "\n", - " # Normalize;\n", - " flux_x[:] /= flux_x.sum()\n", - "\n", - " # LSF\n", - " iix_peak = np.argmax(flux_x)\n", - " xx_tmp_shift = xx_tmp - xx_tmp[iix_peak]\n", - " A, xm, gamma, alpha = LSF_mof(xx_tmp_shift, flux_x)\n", - "\n", - " if ndither == 0:\n", - " # Write it down;\n", - " fm = open('%sl3_nis_%s_%s_s%s_moffat.txt'%(DIR_OUT, filt, grism, id), 'w')\n", - " fm.write('# A x0 gamma alp\\n')\n", - " fm.write('# Moffat function\\n')\n", - " fm.write('%.3f %.3f %.3f %.3f\\n'%(A, xm, gamma, alpha))\n", - " fm.close()\n", - "\n", - "\n", - " #\n", - " # This is for Optimal extraction;\n", - " #\n", - " # Sum along x (disperse) direction\n", - " flux_y = np.zeros(len(sci_rot[:,0]), 'float')\n", - " for ii in range(sci_rot.shape[0]):\n", - " flux_y[ii] = np.sum(sci_rot[ii,:])\n", - "\n", - " \n", - " # Normalize;\n", - " flux_y[:] /= flux_y.sum()\n", - "\n", - "\n", - " # Following Horne;\n", - " flux_disp = np.zeros(x2d, 'float')\n", - " err_disp = np.zeros(x2d, 'float')\n", - " wave_disp = np.zeros(x2d, 'float')\n", - "\n", - " for ii in range(x2d):\n", - " # Mask; \n", - " # 1. DQ array\n", - " # 2. error value\n", - " # 3. CR detection\n", - " mask_tmp = (dq_2d[:,ii] == 0) & (err_2d[:,ii] > 0) \n", - " ivar = 1. / err_2d[:,ii]**2\n", - "\n", - " num = flux_y[:] * data_2d[:,ii] * ivar \n", - " den = flux_y[:]**2 * ivar\n", - " flux_disp[ii] = num[mask_tmp].sum(axis=0)/den[mask_tmp].sum(axis=0)\n", - " err_disp[ii] = np.sqrt(1./den[mask_tmp].sum(axis=0))\n", - " wave_disp[ii] = wave_2d[0,ii]\n", - "\n", - " plt.close()\n", - " con_plot = (wave_disp > 0)\n", - " plt.errorbar(wave_disp[con_plot], flux_disp[con_plot], yerr=err_disp[con_plot])\n", - " plt.ylim(-20, 100)\n", - " plt.show()\n", - "\n", - " # Wirte:\n", - " # Now make it into a Spectrum1D instance.\n", - " file_1d = '%sl3_nis_%s_%s_s%s_ndither%d_1d_opt.fits'%(DIR_OUT, filt, grism, id, ndither)\n", - "\n", - " if wave_disp[1] - wave_disp[0] < 0:\n", - " obs = Spectrum1D(spectral_axis=wave_disp[::-1]*u.um,\n", - " flux=flux_disp[::-1]*u.MJy,\n", - " uncertainty=StdDevUncertainty(err_disp[::-1]), unit='MJy')\n", - " else:\n", - " obs = Spectrum1D(spectral_axis=wave_disp*u.um,\n", - " flux=flux_disp*u.MJy,\n", - " uncertainty=StdDevUncertainty(err_disp), unit='MJy')\n", - "\n", - " obs.write(file_1d, format='tabular-fits', overwrite=True) " - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} From fdae875ceccee422ee171638c2cfc904b28f04bb Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 2 Feb 2024 12:25:04 -0500 Subject: [PATCH 29/62] Changing last modified -> first published --- .../NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb | 2 +- .../NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb | 2 +- notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index 95007b16d..26086ae90 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -25,7 +25,7 @@ "\n", "\n", "**Author**: Camilla Pacifici (cpacifici@stsci.edu) & Rachel Plesha (rplesha@stsci.edu) & Jo Taylor (jotaylor@stsci.edu)
\n", - "**Last modified**: February 2024\n", + "**First Published**: February 2024\n", "\n", "This notebook was inspired by the [JWebbinar session about MAST](https://github.com/spacetelescope/jwebbinar_prep/blob/main/mast_session/Crowded_Field/Crowded_Field.ipynb)." ] diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 9b206b782..8e094a693 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -28,7 +28,7 @@ " - [Manually Editing the Source Catalog](#manual_cat)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", - "**Last modified**: January 2024" + "**First Published**: January 2024" ] }, { diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index 95bb5964d..322525957 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -26,7 +26,7 @@ " - [Final Visualization](#final_visualize)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", - "**Last modified**: February 2024" + "**First Published**: February 2024" ] }, { From a45ba8a7cd703af30f7af94c8856a062ec9e4328 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 2 Feb 2024 12:27:41 -0500 Subject: [PATCH 30/62] Defining WFSS --- .../NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index 26086ae90..df104f1f5 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -5,7 +5,7 @@ "id": "592da8da", "metadata": {}, "source": [ - "# Download WFSS Data\n", + "# Download Wide Field Slittless (WFSS) Data\n", "This notebook uses the python [astroquery.mast Observations](https://astroquery.readthedocs.io/en/latest/mast/mast_obsquery.html) class of the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three [NIRISS filters](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrumentation/niriss-pupil-and-filter-wheels): F115W, F150W, and F200W using both GR150R and GR150C grisms.\n", "\n", "**Use case**: use MAST to download data products.
\n", From 27d6b7a7cd4e002bdc5bd40556ef3af8675641b4 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 5 Feb 2024 18:30:55 -0500 Subject: [PATCH 31/62] Modifying source being shown and only downloading obs 004 --- .../00_niriss_mast_query_data_setup.ipynb | 14 +-- .../01_niriss_wfss_image2_image3.ipynb | 23 ++--- .../02_niriss_wfss_spec2.ipynb | 90 +++++++++---------- 3 files changed, 66 insertions(+), 61 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index df104f1f5..db916e755 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -5,7 +5,7 @@ "id": "592da8da", "metadata": {}, "source": [ - "# Download Wide Field Slittless (WFSS) Data\n", + "# Download Wide Field Slittless Spectroscopy (WFSS) Data\n", "This notebook uses the python [astroquery.mast Observations](https://astroquery.readthedocs.io/en/latest/mast/mast_obsquery.html) class of the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three [NIRISS filters](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrumentation/niriss-pupil-and-filter-wheels): F115W, F150W, and F200W using both GR150R and GR150C grisms.\n", "\n", "**Use case**: use MAST to download data products.
\n", @@ -178,8 +178,8 @@ "source": [ "batch_size = 5 # 5 files at a time maximizes the download speed.\n", "\n", - "# Let's split up our list of files, ``obs_table``, into batches according to our batch size.\n", - "obs_batches = [obs_table[i:i+batch_size] for i in range(0, len(obs_table), batch_size)]\n", + "# Let's split up our list of files, ``obs_id_table``, into batches according to our batch size.\n", + "obs_batches = [obs_id_table[i:i+batch_size] for i in range(0, len(obs_id_table), batch_size)]\n", "print(\"How many batches?\", len(obs_batches))\n", "\n", "single_group = obs_batches[0] # Useful to inspect the files obtained from one group\n", @@ -280,7 +280,7 @@ "id": "e98933e6", "metadata": {}, "source": [ - "From above, we can see that for each exposure name in the observation list (`obs_table`), there are many associated files in the background that need to be downloaded as well. This is why we need to work in batches to download.\n", + "From above, we can see that for each exposure name in the observation list (`obs_id_table`), there are many associated files in the background that need to be downloaded as well. This is why we need to work in batches to download.\n", "\n", "\n", "#### Downloading Data\n", @@ -319,6 +319,10 @@ "source": [ "download_dir = 'data'\n", "\n", + "# make sure the download directory exists; if not, write a new directory\n", + "if not os.path.exists(download_dir):\n", + " os.mkdir(download_dir)\n", + "\n", "# Now let's get the products for each batch of observations, and filter down to only the products of interest.\n", "for index, batch in enumerate(obs_batches):\n", " \n", @@ -348,7 +352,7 @@ " print('Products downloaded:\\n', filtered_products['productFilename'])\n", " \n", " # only downloading the first batch of 5 observations\n", - " #break # comment this out if you want to download everything" + " break # comment this out if you want to download everything, which you'll need to run the rest of the notebooks" ] }, { diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 8e094a693..2c7a073fa 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -481,7 +481,7 @@ " extended_sources = cat[cat['is_extended'] == True]\n", " point_sources = cat[cat['is_extended'] == False]\n", " \n", - " for color, sources in zip(['darkorange', 'black'], [extended_sources, point_sources]):\n", + " for color, sources in zip(['darkred', 'black'], [extended_sources, point_sources]):\n", " # plotting the sources\n", " ax.scatter(sources['xcentroid'], sources['ycentroid'], s=20, facecolors='None', edgecolors=color, alpha=0.9)\n", " \n", @@ -781,7 +781,7 @@ "id": "6ca9e9eb-6420-421a-818b-1faab1db788a", "metadata": {}, "source": [ - "The first step is to decide on a base catalog to match all of the other catalogs to. Here we'll use the the observation 001 F115W filter." + "The first step is to decide on a base catalog to match all of the other catalogs to. Here we'll use the the observation 004 F115W filter." ] }, { @@ -856,7 +856,8 @@ "metadata": {}, "outputs": [], "source": [ - "new_cat_fig = plot_image_and_segmentation_map(cust_image3_i2d, cust_image3_segm, cat_suffix='source-match_cat.ecsv')" + "new_cat_fig = plot_image_and_segmentation_map(cust_image3_i2d, cust_image3_segm, cat_suffix='source-match_cat.ecsv',\n", + " xmin=1500, xmax=2000, ymin=800, ymax=1300)" ] }, { @@ -887,8 +888,8 @@ "metadata": {}, "outputs": [], "source": [ - "# first, look at the current, custom source catalog for one of the filters\n", - "cat_name = np.sort(glob.glob(os.path.join(custom_run_image3, '*source-match_cat.ecsv')))[3]\n", + "# first, look at the current, custom source catalog for the F200W filter\n", + "cat_name = np.sort(glob.glob(os.path.join(custom_run_image3, '*source-match_cat.ecsv')))[2]\n", "cat = Table.read(cat_name)\n", "\n", "print(cat_name)\n", @@ -916,8 +917,8 @@ "outputs": [], "source": [ "# with a known RA/Dec\n", - "desired_ra = 53.16237 \n", - "desired_dec = -27.775\n", + "desired_ra = 53.15437048946369 #53.16237 \n", + "desired_dec = -27.771689847051736 #-27.775\n", "\n", "c = SkyCoord(ra=desired_ra*u.degree, dec=desired_dec*u.degree)\n", "nearest_id, distance_2d, distance_3d = c.match_to_catalog_sky(cat['sky_centroid']) \n", @@ -932,9 +933,9 @@ "metadata": {}, "outputs": [], "source": [ - "# alternatively with a known X/Y pixel location\n", - "known_x = 1580\n", - "known_y = 1400\n", + "# alternatively with a known X/Y pixel location in the F200W image (based on what you've defined cat to be)\n", + "known_x = 1880\n", + "known_y = 1100\n", "\n", "nearest_pos = [np.sqrt((x-known_x)**2 + (y-known_y)**2) for x, y in zip(cat['xcentroid'], cat['ycentroid'])]\n", "\n", @@ -951,7 +952,7 @@ "outputs": [], "source": [ "# alternatively with a known source number\n", - "source = 155\n", + "source = 118\n", "\n", "wh_source = np.where(np.array(cat['label'] == source))[0][0]\n", "\n", diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index 322525957..5f253d92d 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -122,7 +122,11 @@ "source": [ "data_dir = 'data'\n", "custom_run_spec2 = 'custom_spec2'\n", - "custom_run_image3 = os.path.join(data_dir, 'custom_image3_calibrated') # results of custom image3 calibration" + "custom_run_image3 = os.path.join(data_dir, 'custom_image3_calibrated') # results of custom image3 calibration\n", + "\n", + "# if the custom spec2 run directory doesn't exist yet, make it\n", + "if not os.path.exists(os.path.join(data_dir, custom_run_spec2)):\n", + " os.mkdir(os.path.join(data_dir, custom_run_spec2))" ] }, { @@ -178,7 +182,7 @@ "outputs": [], "source": [ "# copy all of the necessary image3 output files\n", - "cats = glob.glob(os.path.join(custom_run_image3, '*source*_cat.ecsv')) # copy both the source-match and source155 catalogs\n", + "cats = glob.glob(os.path.join(custom_run_image3, '*source*_cat.ecsv')) # copy both the source-match and source118 catalogs\n", "segm = glob.glob(os.path.join(custom_run_image3, '*_segm.fits'))\n", "\n", "for image3_file in cats + segm:\n", @@ -543,14 +547,15 @@ " # x1d extension first\n", " x1d_source_ids = np.array([x1d_hdu[ext].header['SOURCEID'] for ext in range(len(x1d_hdu))[1:-1]]) # cut off the 0th and last data extensions\n", " wh_x1d = np.where(x1d_source_ids == source_id)[0][0] + 1 # need to add 1 for the primary header\n", - " if info:\n", - " print(f\"Extension {wh_x1d} in {x1d_hdu[0].header['FILENAME']} contains the data for source {source_id} from our catalog\")\n", " \n", " # look for cal extension, too, but only in the SCI extension; \n", " # fill in with a source ID of -999 for all other extensions to get the right extension value\n", " cal_source_ids = np.array([cal_hdu[ext].header['SOURCEID'] if cal_hdu[ext].header['EXTNAME'] == 'SCI' else -999 for ext in range(len(cal_hdu))[1:-1] ]) \n", " wh_cal = np.where(cal_source_ids == source_id)[0][0] + 1 # need to add 1 for the primary header\n", + "\n", " if info:\n", + " print(f\"All source IDs in x1d file:\\n{x1d_source_ids}\\n\")\n", + " print(f\"Extension {wh_x1d} in {x1d_hdu[0].header['FILENAME']} contains the data for source {source_id} from our catalog\")\n", " print(f\"Extension {wh_cal} in {cal_hdu[0].header['FILENAME']} contains the data for source {source_id} from our catalog\")\n", "\n", " return wh_x1d, wh_cal" @@ -563,12 +568,12 @@ "metadata": {}, "outputs": [], "source": [ - "# Let's look for source 155 that we identified in the previous notebook.\n", - "source_id = 155\n", - "wh_x1d_155, wh_cal_155 = find_source_ext(x1d_hdu, cal_hdu, source_id)\n", + "# Let's look for source 118 that we identified in the previous notebook.\n", + "source_id = 118\n", + "wh_x1d_118, wh_cal_118 = find_source_ext(x1d_hdu, cal_hdu, source_id)\n", "\n", - "x1d_data_155 = x1d_hdu[wh_x1d_155].data \n", - "cal_data_155 = cal_hdu[wh_cal_155].data" + "x1d_data_118 = x1d_hdu[wh_x1d_118].data \n", + "cal_data_118 = cal_hdu[wh_cal_118].data" ] }, { @@ -591,7 +596,7 @@ "rate_data[np.isnan(rate_data)] = 0\n", "\n", "# extraction box parameters from the header of the cal data:\n", - "cal_header = cal_hdu[wh_cal_155].header\n", + "cal_header = cal_hdu[wh_cal_118].header\n", "sx_left = cal_header['SLTSTRT1']\n", "swidth = cal_header['SLTSIZE1']\n", "sx_right = cal_header['SLTSTRT1'] + swidth\n", @@ -624,11 +629,11 @@ "metadata": {}, "outputs": [], "source": [ - "def plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file):\n", + "def plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file, source_id):\n", " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 5))\n", " \n", " # plot the cal image\n", - " ax1.imshow(cal_data, origin='lower', vmin=0, vmax=np.nanmean(cal_data), aspect='auto')\n", + " ax1.imshow(cal_data, origin='lower', vmin=0, vmax=np.nanmax(cal_data)*.01, aspect='auto')\n", " ax1.set_title(os.path.basename(cal_file))\n", " \n", " # plot the spectrum\n", @@ -641,8 +646,10 @@ " edge_buffer = int(len(flux) * .25)\n", " max_flux = np.nanmax(flux[edge_buffer:edge_buffer*-1])\n", " ax2.set_ylim(0, max_flux+(max_flux*0.1)) # cutting the flux of the edges & adding 10% buffer to the limits\n", + " ax2.set_xlabel('Wavelength (Microns)')\n", + " ax2.set_ylabel('Flux')\n", " \n", - " plt.suptitle(f\"{fits.getval(cal_file, 'FILTER')} {fits.getval(cal_file, 'PUPIL')}\")" + " plt.suptitle(f\"{fits.getval(cal_file, 'FILTER')} {fits.getval(cal_file, 'PUPIL')} Source {source_id}\")" ] }, { @@ -652,7 +659,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_cutout_and_spectrum(cal_data_155, x1d_data_155, cal_file, x1d_file)" + "plot_cutout_and_spectrum(cal_data_118, x1d_data_118, cal_file, x1d_file, source_id)" ] }, { @@ -727,11 +734,18 @@ "metadata": {}, "outputs": [], "source": [ - "# look at the minimum and maximum magnitudes\n", + "source_match_cats = np.sort(glob.glob('*source-match_cat.ecsv'))\n", + "source_match_cat = Table.read(source_cat)\n", + "\n", + "# look at the possible magnitude ranges to look at\n", "mags = cat['isophotal_vegamag']\n", "min_vegmag = mags.min()\n", "max_vegmag = mags.max()\n", - "print(f\"Magnitude range: {min_vegmag} - {max_vegmag}\")" + "print(f\"Magnitude range: {min_vegmag} - {max_vegmag}\")\n", + "\n", + "# source 118 should have a Vega mag of ~21.68\n", + "source_id = 118\n", + "print(f\"Magnitude for source in previous notebook: source {source_id} : {source_match_cat[source_match_cat['label'] == source_id]['isophotal_vegamag'][0]}\")" ] }, { @@ -742,9 +756,9 @@ "outputs": [], "source": [ "# find the catalog for how many stars are between a specific magnitude \n", - "min_mag = 20.5\n", - "max_mag = 21\n", - "mag_cat = cat[(cat['isophotal_vegamag'] >= min_mag) & (cat['isophotal_vegamag'] <= max_mag)]\n", + "min_mag = 21.6\n", + "max_mag = 21.7\n", + "mag_cat = source_match_cat[(source_match_cat['isophotal_vegamag'] >= min_mag) & (source_match_cat['isophotal_vegamag'] <= max_mag)]\n", "\n", "mag_cat['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']" ] @@ -781,7 +795,7 @@ "metadata": {}, "outputs": [], "source": [ - "for cat_name in glob.glob('*source-match_cat.ecsv'):\n", + "for cat_name in source_match_cats:\n", "\n", " if cat_name == source_cat:\n", " # the base one has already been saved\n", @@ -871,16 +885,6 @@ "Now when we calibrate everything, it should take a lot less time because there are a limited number of sources." ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "f60f8622-857c-43de-8f18-f251e8e85aa4", - "metadata": {}, - "outputs": [], - "source": [ - "ls new_catalog_calibrated/jw02079004001_03101_00001_nis_x1d.fits" - ] - }, { "cell_type": "code", "execution_count": null, @@ -925,9 +929,7 @@ "cell_type": "code", "execution_count": null, "id": "94aec550-ea1f-4215-8965-bcd2ac408bfd", - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "#### Explore the new Data\n", @@ -935,7 +937,7 @@ "\n", "# get a list of all of the source IDs from the first file to look at for this example\n", "sources = [fits.getval(x1ds[0], 'SOURCEID', ext=ext) for ext in range(len(fits.open(x1ds[0])))[1:-1]]\n", - "source_id = 245\n", + "source_id = 118\n", "\n", "# plot each x1d/cal file\n", "for i, x1d_file in enumerate(x1ds):\n", @@ -951,7 +953,7 @@ " x1d_data = x1d_hdu[wh_x1d].data \n", " cal_data = cal_hdu[wh_cal].data\n", "\n", - " plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file)" + " plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file, source_id)" ] }, { @@ -966,19 +968,17 @@ "cell_type": "code", "execution_count": null, "id": "aa3b5777-4368-41fa-942f-c3a6f003d1a3", - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ - "x1d_file = x1ds[0]\n", + "x1d_file = x1ds[2]\n", "cal_file = x1d_file.replace('x1d.fits', 'cal.fits')\n", "with fits.open(x1d_file) as x1d_hdu, fits.open(cal_file) as cal_hdu:\n", "\n", " for ext in range(len(x1d_hdu))[1:-1]:\n", "\n", " source_id = x1d_hdu[ext].header['SOURCEID']\n", - " print(source_id)\n", + "\n", " try:\n", " wh_x1d, wh_cal = find_source_ext(x1d_hdu, cal_hdu, source_id, info=False)\n", " except IndexError:\n", @@ -988,7 +988,7 @@ " x1d_data = x1d_hdu[wh_x1d].data \n", " cal_data = cal_hdu[wh_cal].data\n", " \n", - " plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file)" + " plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file, source_id)" ] }, { @@ -1022,7 +1022,7 @@ "rate_data[np.isnan(rate_data)] = 0\n", "\n", "# plot the rate file and the extraction box\n", - "ax1.imshow(rate_data, origin='lower', vmin=0, vmax=np.nanmean(cal_data)*0.2, aspect='auto')\n", + "ax1.imshow(rate_data, origin='lower', vmin=0, vmax=np.nanmax(rate_data)*0.01, aspect='auto')\n", "\n", "with fits.open(x1d_file) as x1d_hdu, fits.open(cal_file) as cal_hdu:\n", "\n", @@ -1048,9 +1048,9 @@ " sheight = cal_header['SLTSIZE2']\n", " sy_top = cal_header['SLTSTRT2'] + sheight\n", " \n", - " rectangle = patches.Rectangle((sx_left, sy_bottom), swidth, sheight, edgecolor='darkorange', facecolor=\"None\", linewidth=1)\n", + " rectangle = patches.Rectangle((sx_left, sy_bottom), swidth, sheight, edgecolor='darkred', facecolor=\"None\", linewidth=1)\n", " ax1.add_patch(rectangle)\n", - " ax1.text(sx_left, sy_top+10, source_id, fontsize=14, color='darkorange')\n", + " ax1.text(sx_left, sy_top+10, source_id, fontsize=14, color='darkred')\n", " \n", " ax1.set_title(f\"{os.path.basename(x1d_file).split('_nis')[0]}: {cal_hdu[0].header['FILTER']} {cal_hdu[0].header['PUPIL']}\")\n", "\n", @@ -1067,7 +1067,7 @@ "extended_sources = cat[cat['is_extended'] == True]\n", "point_sources = cat[cat['is_extended'] == False]\n", "\n", - "for color, sources in zip(['darkorange', 'black'], [extended_sources, point_sources]):\n", + "for color, sources in zip(['darkred', 'black'], [extended_sources, point_sources]):\n", " # plotting the sources\n", " ax2.scatter(sources['xcentroid'], sources['ycentroid'], s=40, facecolors='None', edgecolors=color, alpha=0.9, lw=2)\n", "\n", From 6f3abe66ee14d3c4a800ed3df4cf05530de8118e Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 8 Feb 2024 19:39:17 -0500 Subject: [PATCH 32/62] Addressing Cami's comments so far for notebooks 00 and 01 --- .../00_niriss_mast_query_data_setup.ipynb | 28 ++++++++++--- .../01_niriss_wfss_image2_image3.ipynb | 42 +++++++++++++------ .../02_niriss_wfss_spec2.ipynb | 36 ++++++++++------ 3 files changed, 76 insertions(+), 30 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index db916e755..9dd544fec 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -311,18 +311,26 @@ { "cell_type": "code", "execution_count": null, - "id": "616ff15c", - "metadata": { - "scrolled": true - }, + "id": "21c50976-a56a-4cd2-9cd6-11f9266b61bb", + "metadata": {}, "outputs": [], "source": [ "download_dir = 'data'\n", "\n", "# make sure the download directory exists; if not, write a new directory\n", "if not os.path.exists(download_dir):\n", - " os.mkdir(download_dir)\n", - "\n", + " os.mkdir(download_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "616ff15c", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ "# Now let's get the products for each batch of observations, and filter down to only the products of interest.\n", "for index, batch in enumerate(obs_batches):\n", " \n", @@ -355,6 +363,14 @@ " break # comment this out if you want to download everything, which you'll need to run the rest of the notebooks" ] }, + { + "cell_type": "markdown", + "id": "31ff8769-1240-452d-b84c-eca69d53a13f", + "metadata": {}, + "source": [ + "If continuing on with the WFSS notebooks, let's double check that we've downloaded all of the files that we need for the remaining notebooks. There should be 143 files downloaded. " + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 2c7a073fa..c21a522a9 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -10,7 +10,7 @@ "In this run, we are starting from the rate files which have already been calibrated with the detector1 pipeline. This will save time as we do not need to edit any of the steps being performed as part of detector1. Therefore, the first calibration that should be done as part of a WFSS run is to run the rate files direct images through the Image2 and Image3 steps of the JWST pipeline. This includes creating a source catalog, which most likely will need to be adjusted from the pipeline default values. **Not having a good source catalog will result in non optimal extraction of sources in the dispersed, WFSS, images.**\n", "\n", "**Use case**: The default parameters for the pipeline do not extract the expected sources, so a custom parameters need to be set to obtain new combined image and source catalog.
\n", - "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", + "**Data**: JWST/NIRISS images and spectra from program 2079 observation 004. This should be stored in a single directory `data`, and can be downloaded from the previous notebook, 00_niriss_mast_query_data_setup.ipynb.
\n", "**Tools**: astropy, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, warnings
\n", "**Cross-instrument**: NIRISS
\n", "\n", @@ -40,6 +40,26 @@ "## Imports & Data Setup" ] }, + { + "cell_type": "markdown", + "id": "e3ccd6a3-e781-4d7f-8120-fa142723a6d9", + "metadata": {}, + "source": [ + "[CRDS Documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/user_documentation/reference_files_crds.html#crds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bf2ea52-2568-44cf-8074-eec3b5b76cc0", + "metadata": {}, + "outputs": [], + "source": [ + "# Update the CRDS path to your local directory\n", + "%env CRDS_PATH=crds_cache\n", + "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" + ] + }, { "cell_type": "code", "execution_count": null, @@ -68,16 +88,6 @@ "from jwst.pipeline import Image3Pipeline" ] }, - { - "cell_type": "raw", - "id": "42888b91-e010-4ed2-b101-f586668a2235", - "metadata": {}, - "source": [ - "# Update the CRDS path to your local directory\n", - "%env CRDS_PATH= (/grp/crds/jwst/)\n", - "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" - ] - }, { "cell_type": "markdown", "id": "fb035a4d-e63c-4125-b083-462e3353e49f", @@ -96,6 +106,14 @@ "print('jwst:', jwst.__version__)" ] }, + { + "cell_type": "markdown", + "id": "feb25e11-bc16-44b8-830c-d34f12e2dcf7", + "metadata": {}, + "source": [ + "The data directory, `data_dir` here should contain all of the association and rate files in a single, flat directory. `default_run_image3` and `custom_run_image3` are directories that we will use later for our calibrated data. They are separated so that we can compare the two outputs." + ] + }, { "cell_type": "code", "execution_count": null, @@ -339,7 +357,7 @@ "tags": [] }, "source": [ - "Image3 is where we can make some adjustments to obtain a better output source catalog. The `cal.fits` files will be calibrated into a single combined `i2d.fits` image. More information about the steps performed in the Image3 part of the pipeline can be found in the [Image3 pipeline documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_image3.html).\n", + "In Image3, the `cal.fits` files will be calibrated into a single combined `i2d.fits` image. The Image3 step is also where the final source catalog is created, so we can change some input paramters to obtain a more refined output source catalog. This is done in the [Custom Imaging Pipeline Run](#custom) section. However, we will first calibrate the data with the default parameters. More information about the steps performed in the Image3 part of the pipeline can be found in the [Image3 pipeline documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_image3.html).\n", "\n", "**Note: Image3 can take a while to run**" ] diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index 5f253d92d..40bf45000 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -10,7 +10,7 @@ "The dispersed images for WFSS will be run through the [spec2 pipeline](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_spec2.html) after the image2 and image3 pipelines have been run on the corresponding direct images. As mentioned in the previous notebook, it is extremely helpful for this step if you have a source catalog that only includes the sources that you would like to look at. Any additional sources will take extra time to calibrate.\n", "\n", "**Use case**: After creating a custom source catalog, spec2 should be run on the dispersed WFSS images.
\n", - "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", + "**Data**: JWST/NIRISS images and spectra from program 2079 obs 004. This should be stored in a single directory `data`, and can be downloaded from the notebook 00_niriss_mast_query_data_setup.ipynb.
\n", "**Tools**: astropy, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, shutil
\n", "**Cross-instrument**: NIRISS
\n", "\n", @@ -29,6 +29,26 @@ "**First Published**: February 2024" ] }, + { + "cell_type": "markdown", + "id": "ab693165-555f-41b3-8135-1155da31393f", + "metadata": {}, + "source": [ + "[CRDS Documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/user_documentation/reference_files_crds.html#crds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "974ab7ce-d1dd-4bcc-b405-25c6dd683a25", + "metadata": {}, + "outputs": [], + "source": [ + "# Update the CRDS path to your local directory\n", + "%env CRDS_PATH=crds_cache\n", + "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" + ] + }, { "cell_type": "markdown", "id": "b28e541b", @@ -64,16 +84,6 @@ "from jwst import datamodels" ] }, - { - "cell_type": "raw", - "id": "41c4c7c2-7914-4f5f-ad5f-4b876bc63555", - "metadata": {}, - "source": [ - "# Update the CRDS path to your local directory\n", - "%env CRDS_PATH= (/grp/crds/jwst/)\n", - "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" - ] - }, { "cell_type": "markdown", "id": "64b6e8d7-64a0-4a3b-9776-29b181452ff9", @@ -100,7 +110,9 @@ "source": [ "#### Data Setup\n", "\n", - "For spec2, we need the rate files that we downloaded as well as the segmentation maps and source catalogs from image3. Because of that, we will create a new directory, change into it, and copy the relevant data over. \n", + "The data directory, `data_dir` should contain all of the association and rate files in a single, flat directory. `custom_run_image3` should match the output directory for the custom image3 calibration in the notebook 01_niriss_wfss_image2_image3.ipynb.\n", + "\n", + "For spec2, we need the rate files that we downloaded as well as the segmentation maps and source catalogs from image3. Because of that, we will create a new directory (defined by `custom_run_spec2`), change into it, and copy the relevant data over. \n", "\n", "*For a regular workflow, it is likely easier to leave all files in a single directory compared to the multiple directories we make here.*" ] From 00ce28e6290df433a5c2c04357243aefad41623f Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 9 Feb 2024 13:02:40 -0500 Subject: [PATCH 33/62] Adding more description of what typical observations & this one look like from Steph's comments --- .../00_niriss_mast_query_data_setup.ipynb | 88 +++++++++++++++++-- 1 file changed, 79 insertions(+), 9 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index 9dd544fec..6f7f85b60 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -6,11 +6,11 @@ "metadata": {}, "source": [ "# Download Wide Field Slittless Spectroscopy (WFSS) Data\n", - "This notebook uses the python [astroquery.mast Observations](https://astroquery.readthedocs.io/en/latest/mast/mast_obsquery.html) class of the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three [NIRISS filters](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrumentation/niriss-pupil-and-filter-wheels): F115W, F150W, and F200W using both GR150R and GR150C grisms.\n", + "This notebook uses the python [astroquery.mast Observations](https://astroquery.readthedocs.io/en/latest/mast/mast_obsquery.html) class of the [MAST API](https://mast.stsci.edu/api/v0/) to query specific data products of a specific program. We are looking for NIRISS imaging and WFSS files of the [NGDEEP program](https://www.stsci.edu/jwst/phase2-public/2079.pdf) (ID 2079). The observations are in three [NIRISS filters](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrumentation/niriss-pupil-and-filter-wheels): F115W, F150W, and F200W using both GR150R and GR150C [grisms](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrumentation/niriss-gr150-grisms). A [WFSS observation sequence](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-observing-strategies/niriss-wfss-recommended-strategies) typically consists of a direct image followed by a grism observation in the same blocking filter to help identify the sources in the field. In program 2079, the exposure sequence follows the pattern: direct image -> GR150R -> direct image -> GR150C -> direct image.\n", "\n", "**Use case**: use MAST to download data products.
\n", - "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: astropy, astroquery, glob, numpy, os, pandas, (yaml)
\n", + "**Data**: JWST/NIRISS images and spectra from program 2079 observation 004.
\n", + "**Tools**: astropy, astroquery, glob, matplotlib, numpy, os, pandas, (yaml)
\n", "**Cross-instrument**: all
\n", "\n", "**Content**\n", @@ -21,7 +21,7 @@ "- [Filter and Download Products](#filter)\n", " - [Filtering Data Before Downloading](#filter_data)\n", " - [Downloading Data](#downloading)\n", - "- [Create a Keyword Dataframe](#keyword)\n", + "- [Inspect Downloaded Data](#inspect)\n", "\n", "\n", "**Author**: Camilla Pacifici (cpacifici@stsci.edu) & Rachel Plesha (rplesha@stsci.edu) & Jo Taylor (jotaylor@stsci.edu)
\n", @@ -50,6 +50,7 @@ "from astropy.coordinates import SkyCoord\n", "from astropy.io import fits\n", "from astroquery.mast import Observations\n", + "from matplotlib import pyplot as plt\n", "import numpy as np\n", "import os\n", "import glob\n", @@ -131,7 +132,7 @@ "source": [ "\n", "#### Search with Observation ID\n", - "The observation ID (obs_id) allows for flexibility of searching by the proposal ID and the observation ID because of how the JWST filenames are structured. More information about the JWST file naming conventions can be found at: https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/file_naming.html.\n", + "The observation ID (obs_id) allows for flexibility of searching by the proposal ID and the observation ID because of how the JWST filenames are structured. More information about the JWST file naming conventions can be found at: https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/file_naming.html. For the purposes of this notebook series, we will use only one of the two observations (004) in program 2079.\n", "\n", "Additionally, there is flexibility using wildcards inside of the search criteria. For example, instead of specifying both \"NIRISS/IMAGE\" and \"NIRISS/WFSS\", we can specify \"NIRISS*\", which picks up both file modes. The wildcard also works within the obs_id, so we do not have to list all of the different IDs." ] @@ -286,10 +287,14 @@ "#### Downloading Data\n", "To actually download the products, provide ``Observations.download_products()`` with a list of the filtered products. \n", "\n", + "Typically, adjustments aren't needed to the [detector1 pipeline](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_detector1.html), so we can start with the outputs from detector1, the rate files, rather than the uncal files. Because of this, we only need to download the rate and association files. If you need to rerun the detector1 pipeline, `productSubGroupDescription` and `calib_level` will need to be adjusted in the `Observations.filter_products` call to download the uncal files instead.\n", + "\n", "If the data are proprietary, you may also need to set up your API token. **NEVER** commit your token to a public repository. An alternative is to create a separate configuration file (config_file.yaml) that is readable only to you and has a key: 'mast_token' : _API token_\n", "\n", "To make create a new API token visit to following link: \n", - " https://auth.mast.stsci.edu/token?suggested_name=Astroquery&suggested_scope=mast:exclusive_access" + " https://auth.mast.stsci.edu/token?suggested_name=Astroquery&suggested_scope=mast:exclusive_access\n", + "\n", + "*Note that a developmental version of astroquery is currently required to have the call `flat=True` when downloading the data. If you prefer to not use a developmental version, remove that line from the call, download the data, and move all of the files in all downloaded subfolders into the single directory as defined by the `download_dir` variable.*" ] }, { @@ -387,8 +392,8 @@ "id": "e840d7c4-3643-4152-bd04-bfdcc150cb3e", "metadata": {}, "source": [ - "\n", - "## Create a Keyword Dataframe\n", + "\n", + "## Inspect Downloaded Data\n", "\n", "The purpose of this function is to have a better idea of what data are available to you. Additionally, you will be able to use this dataframe to select specific files that match the mode you would like to take a closer look at." ] @@ -403,7 +408,7 @@ "ratefile_datadir = 'data/'\n", "\n", "# first look for all of the rate files you have downloaded\n", - "rate_files = glob.glob(os.path.join(ratefile_datadir, \"*rate.fits\"))\n", + "rate_files = np.sort(glob.glob(os.path.join(ratefile_datadir, \"*rate.fits\")))\n", "\n", "for file_num, ratefile in enumerate(rate_files):\n", "\n", @@ -444,6 +449,71 @@ "rate_df" ] }, + { + "cell_type": "markdown", + "id": "4f49059b-cc92-448f-aca1-b5af87b0c208", + "metadata": {}, + "source": [ + "In particular, let's look at the observation sequence these rate files follow. We have sorted the filenames above, so they should already be in time order in the dataframe. \n", + "\n", + "`FILTER = CLEAR` indicates a direct image while `FILTER=GR150R` or `FILTER=GR150C` indicates a dispersed image. PUPIL is the blocking filter used. The first 14 exposures make up the first sequence set of direct image -> grism -> direct image -> grism. There are also multiple dither positions for the dispersed images and the direct images. The multiple direct image dithers will be combined in image3, while the multiple dispersed images can be combined in spec3. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ccd23e56-4441-4ad0-ac90-68358802133d", + "metadata": {}, + "outputs": [], + "source": [ + "rate_df[['EXPSTART', 'FILTER', 'PUPIL', 'PATT_NUM', 'XOFFSET', 'YOFFSET']].head(14)" + ] + }, + { + "cell_type": "markdown", + "id": "38503db6-e37b-4237-b518-6b1a15cefd66", + "metadata": {}, + "source": [ + "Shown below are the first 14 rate files to give an idea of the above sequence visually. Grid lines are shown as a visual guide for any dithers that are made." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a285064-b1ee-4ead-8890-4d20f1ba7865", + "metadata": {}, + "outputs": [], + "source": [ + "# plot set up\n", + "fig = plt.figure(figsize=(20, 35))\n", + "cols = 3\n", + "rows = int(np.ceil(14 / cols))\n", + "\n", + "# loop over the first 14 rate files and plot them\n", + "for plt_num, rf in enumerate(rate_df[0:14]['FILENAME']):\n", + "\n", + " # determine where the subplot should be\n", + " xpos = (plt_num%40) % cols\n", + " ypos = ((plt_num%40) // cols) # // to make it an int.\n", + "\n", + " # make the subplot\n", + " ax = plt.subplot2grid((rows, cols), (ypos, xpos))\n", + "\n", + " # open the data and plot it\n", + " with fits.open(rf) as hdu:\n", + " data = hdu[1].data\n", + " data[np.isnan(data)] = 0 # filling in nan data with 0s to help with the matplotlib color scale.\n", + " \n", + " ax.imshow(data, vmin=0, vmax=1.5, origin='lower')\n", + "\n", + " # adding in grid lines as a visual aid\n", + " for gridline in [500, 1000, 1500]:\n", + " ax.axhline(gridline, color='black', alpha=0.5)\n", + " ax.axvline(gridline, color='black', alpha=0.5)\n", + "\n", + " ax.set_title(f\"#{plt_num+1}: {hdu[0].header['FILTER']} {hdu[0].header['PUPIL']} Dither{hdu[0].header['PATT_NUM']}\")" + ] + }, { "cell_type": "markdown", "id": "8c0e1d71", From c06c40c87901f5484939157a940c801edf654f8b Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 9 Feb 2024 15:00:56 -0500 Subject: [PATCH 34/62] text edits thanks to Steph's suggestions --- .../01_niriss_wfss_image2_image3.ipynb | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index c21a522a9..a32d80e16 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -9,7 +9,7 @@ "\n", "In this run, we are starting from the rate files which have already been calibrated with the detector1 pipeline. This will save time as we do not need to edit any of the steps being performed as part of detector1. Therefore, the first calibration that should be done as part of a WFSS run is to run the rate files direct images through the Image2 and Image3 steps of the JWST pipeline. This includes creating a source catalog, which most likely will need to be adjusted from the pipeline default values. **Not having a good source catalog will result in non optimal extraction of sources in the dispersed, WFSS, images.**\n", "\n", - "**Use case**: The default parameters for the pipeline do not extract the expected sources, so a custom parameters need to be set to obtain new combined image and source catalog.
\n", + "**Use case**: The default parameters for the pipeline do not extract the expected sources, so custom parameters need to be set to obtain new combined image and source catalog.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079 observation 004. This should be stored in a single directory `data`, and can be downloaded from the previous notebook, 00_niriss_mast_query_data_setup.ipynb.
\n", "**Tools**: astropy, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, warnings
\n", "**Cross-instrument**: NIRISS
\n", @@ -134,6 +134,18 @@ "The association files expect that 1) all of the data are in the same directory and 2) that you are performing the pipeline call also in that directory. Because of that, we need to change into the data directory to run the imaging pipelines." ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "3d348ad9-4344-4a03-a58d-344357eeea28", + "metadata": {}, + "outputs": [], + "source": [ + "# From the csv file we created earlier, find a list of all of the grism observations we will want to calibrate with spec2\n", + "listrate_file = 'list_ngdeep_rate.csv'\n", + "rate_df = pd.read_csv(listrate_file)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -189,7 +201,9 @@ "metadata": {}, "source": [ "#### Looking in a Level 2 Imaging Association File\n", - "First, take a look inside the association (ASN) files to better understand everything that is contained in them." + "First, take a look inside the association (ASN) files to better understand everything that is contained in them.\n", + "\n", + "For image2 association files, there should be one asn file for each dither position in an observing sequence which is set by the [exposure strategy](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-observing-strategies/niriss-wfss-recommended-strategies). In this case, that should match the number of direct images (`FILTER=CLEAR`) in `rate_df` because each direct image is at a unique dither position (XOFFSET, YOFFSET) within an observing sequence. For this program and observation, there is one direct image before a grism change with only one dither, another direct image with four dithers between the change in grisms, and a direct image at the end of a sequence with three dithers. This leads to a total of eight images per observing sequence, with five observing sequences in the observation using the blocking filters F115W -> F115W -> F150W -> F150W -> F200W." ] }, { @@ -200,7 +214,10 @@ "outputs": [], "source": [ "image2_asns = glob.glob('*image2*asn*.json')\n", - "print(len(image2_asns), 'Image2 ASN files')" + "print(len(image2_asns), 'Image2 ASN files') # there should be 40 asn files for image2\n", + "\n", + "# the number of association files should match the number of rate files\n", + "print(len(rate_df[rate_df['FILTER'] == 'CLEAR']), 'Direct Image rate files')" ] }, { @@ -225,7 +242,7 @@ "1. From `asn_type` and `asn_rule`, we can see that this is an image2 association\n", "2. From `degraded_status` we can see that there are no exposures to not be included in the calibration.\n", "3. From `constraints`, we can see this is not a time series observation (TSO), the observation is part of program 2079, observed with NIRISS with the CLEAR (i.e. imaging for WFSS) and F150W filters.\n", - "4. From `products` we can see there is only one exposure associated" + "4. From `products` we can see there is only one exposure associated. This is typical for image2 where there is usually only one exposure per dither per observing sequence." ] }, { @@ -293,7 +310,7 @@ "metadata": {}, "source": [ "#### Looking in a Level 3 Association File\n", - "The contents are quite similar to image2, but notice now that there are many more members that are associated together, and they use the `cal.fits` files from image2." + "The contents are quite similar to image2, but notice now that there are many more members that are associated together, and they use the individual pointing cal files from image2. Image3 resamples and combines images of the same blocking filter (PUPIL for NIRISS WFSS) from all dither and observing sequences to form a single image, which leads to fewer image3 association files." ] }, { @@ -304,7 +321,11 @@ "outputs": [], "source": [ "image3_asns = glob.glob('*image3*asn*.json')\n", - "print(len(image3_asns), 'Image3 ASN files')" + "print(len(image3_asns), 'Image3 ASN files') # there should be 3 image3 association files\n", + "\n", + "# the number of image3 association files should match the number of unique blocking filters used\n", + "uniq_filters = np.unique(rate_df[rate_df['FILTER'] == 'CLEAR']['PUPIL'])\n", + "print(f\"{len(uniq_filters)} unique filters used: {uniq_filters}\")\n" ] }, { @@ -357,7 +378,7 @@ "tags": [] }, "source": [ - "In Image3, the `cal.fits` files will be calibrated into a single combined `i2d.fits` image. The Image3 step is also where the final source catalog is created, so we can change some input paramters to obtain a more refined output source catalog. This is done in the [Custom Imaging Pipeline Run](#custom) section. However, we will first calibrate the data with the default parameters. More information about the steps performed in the Image3 part of the pipeline can be found in the [Image3 pipeline documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_image3.html).\n", + "In Image3, the `cal.fits` individual pointing files will be calibrated into a single combined `i2d.fits` image. The Image3 step is also where the final source catalog is created, so we can change some input paramters to obtain a more refined output source catalog. This is done below in the [Custom Imaging Pipeline Run](#custom) section. However, we will first calibrate the data with the default parameters. More information about the steps performed in the Image3 part of the pipeline can be found in the [Image3 pipeline documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_image3.html).\n", "\n", "**Note: Image3 can take a while to run**" ] @@ -457,7 +478,9 @@ "source": [ "The segmentation maps are also a product of the Image3 pipeline, and they are used the help determine the source catalog. Let's take a look at those to ensure we are happy with what it is defining as a source.\n", "\n", - "In the segmentation map, each yellow blob should correspond to a physical target. There are cases where sources can be blended, in which case the parameters for making the semgentation map and source catalog should be changed. An example of this can be seen below in the observation 004 F200W filter image where two galaxies have been blended into one source. This is discussed in more detail in [Custom Imaging Pipeline Run](#custom)." + "In the segmentation map, each blue blob should correspond to a physical target. There are cases where sources can be blended, in which case the parameters for making the semgentation map and source catalog should be changed. An example of this can be seen below in the observation 004 F200W filter image where two galaxies at ~(1600, 1300) have been blended into one source. This is discussed in more detail below in [Custom Imaging Pipeline Run](#custom). \n", + "\n", + "*Note that because of the [filter offset difference](https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrumentation/niriss-gr150-grisms#NIRISSGR150Grisms-Blockingfilters) the field of views for the shown in the cutouts below differs for each filter.*" ] }, { @@ -606,7 +629,9 @@ "source": [ "Try editing a few parameters and compare the outcomes to the default run above, at first for a single file.\n", "\n", - "When we call the image3 pipeline, we can add modifications to a specific step of the pipeline. In this case we're going to edit the `source_catalog` and `tweakreg` steps of the pipeline." + "When we call the image3 pipeline, we can add modifications to a specific step of the pipeline. In this case we're going to edit the `source_catalog` and `tweakreg` steps of the pipeline. An explanation of the different parameters to tweak can be found in the further information below, while the default values are a combination of both the default pipeline values listed in there, and the parameter reference files that are supplied.\n", + "- [source_catalog Further Information](https://jwst-pipeline.readthedocs.io/en/latest/api/jwst.source_catalog.SourceCatalogStep.html)\n", + "- [tweakreg Futher Information](https://jwst-pipeline.readthedocs.io/en/latest/api/jwst.tweakreg.tweakreg_step.TweakRegStep.html)" ] }, { @@ -780,7 +805,7 @@ "id": "191a12fb-0989-46d2-8e62-792cf3b23f4a", "metadata": {}, "source": [ - "In the above cases, we have modified the results using the pipeline directly. It might be the case that perhaps you want to use someone else's custom source catalog, or modify the source catalog even further from what was output by the pipeline. In these cases, we will then need to modify the spec2 ASN files to point to the new source catalog, which will be discussed in the spec2 notebook. Additionally, it can be useful to match all of the source IDs across the different catalogs. In this case, there are four different catalogs created by the pipeline that identify the same RA/Dec or X/Y pixel location as a different source ID, so we will edit those to have the same source ID values across the catalogs. These extra steps aren't always necessary, but could be helpful in analyses of NIRISS WFSS data." + "In the above cases, we have modified the results using the pipeline directly. It might be the case that perhaps you want to use someone else's custom source catalog, or modify the source catalog even further from what was output by the pipeline. In these cases, we will then need to modify the spec2 ASN files to point to the new source catalog, which will be discussed in the spec2 notebook. Additionally, it can be useful to match all of the source IDs across the different catalogs. In this case, there are three different catalogs created by the pipeline that identify the same RA/Dec or X/Y pixel location as a different source ID, so we will edit those to have the same source ID values across the catalogs. These extra steps aren't always necessary, but could be helpful in analyses of NIRISS WFSS data." ] }, { From d57971ff6be3ef0af0dc2cb253375a5baef2f612 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 9 Feb 2024 18:02:27 -0500 Subject: [PATCH 35/62] adding in more comments from Steph --- .../02_niriss_wfss_spec2.ipynb | 54 +++++++++++++------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index 40bf45000..ddc4ab31f 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -29,6 +29,15 @@ "**First Published**: February 2024" ] }, + { + "cell_type": "markdown", + "id": "b28e541b", + "metadata": {}, + "source": [ + "\n", + "## Imports & Data Setup" + ] + }, { "cell_type": "markdown", "id": "ab693165-555f-41b3-8135-1155da31393f", @@ -49,15 +58,6 @@ "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" ] }, - { - "cell_type": "markdown", - "id": "b28e541b", - "metadata": {}, - "source": [ - "\n", - "## Imports & Data Setup" - ] - }, { "cell_type": "code", "execution_count": null, @@ -265,7 +265,7 @@ "\n", "### Spec2 Association Files\n", "\n", - "As with the imaging part of the pipeline, there are association files for spec2. These are a bit more complex in that they need to have the science (WFSS) data, direct image, source catalog, and segmentation map included as members. " + "As with the imaging part of the pipeline, there are association files for spec2. These are a bit more complex in that they need to have the science (WFSS) data, direct image, source catalog, and segmentation map included as members. For the science data, the rate files are used as inputs, similarly to image2. Also like image2, there should be one asn file for each dispersed image dither position in an observing sequence. In this case, that should match the number of rate files where `FILTER=GR150R` or `FILTER=GR150C`. For this program and observation, there are three dithers per grism, and both GR150R and GR150C are used, totaling six exposures per observing sequence with five observing sequences in the observation using the blocking filters F115W -> F115W -> F150W -> F150W -> F200W." ] }, { @@ -284,7 +284,9 @@ "outputs": [], "source": [ "spec2_asns = np.sort(glob.glob('*spec2*asn*.json'))\n", - "print(len(spec2_asns), 'Spec2 ASN files')" + "print(len(spec2_asns), 'Spec2 ASN files')\n", + "# number of asn files should match the number of dispersed images -- 30 for obs 004\n", + "print(len(rate_df[(rate_df['FILTER'] == 'GR150R') | (rate_df['FILTER'] == 'GR150C')]), 'Dispersed image rate files')" ] }, { @@ -545,7 +547,7 @@ "\n", "### Find a Source in the Spec2 Data\n", "\n", - "Each extension of the cal and x1d files has a source ID in the header. With newer versions of the JWST pipeline, these values should match with the values in the source catalog." + "Each extension of the cal and x1d files has a source ID in the header. These values should match with the values in the source catalog." ] }, { @@ -593,7 +595,9 @@ "id": "a362fa9f-ae33-4dcc-954c-471a80b41d37", "metadata": {}, "source": [ - "First, let's look at the extraction box as it appears on the rate image. This will help us get a global feel for how much we can trust the pipeline's extraction." + "First, let's look at the extraction box as it appears on the rate image. This will help us get a global feel for how much we can trust the pipeline's extraction. \n", + "\n", + "*Note: In this example, the extraction box isn't fully cenetered around the spectrum. There is an ongoing calibration effort to better account for difference in the spectral trace shape across the NIRISS detector. Updates about the status of this calibration can be found on the [NIRISS jdox](https://jwst-docs.stsci.edu/jwst-calibration-pipeline-caveats/jwst-wide-field-slitless-spectroscopy-pipeline-caveats#JWSTWideFieldSlitlessSpectroscopyPipelineCaveats-NIRISSWFSS)*" ] }, { @@ -634,6 +638,16 @@ "plt.suptitle(f\"{cal_hdu[0].header['FILTER']} {cal_hdu[0].header['PUPIL']}\")" ] }, + { + "cell_type": "markdown", + "id": "22a7c98b-0a1b-4d1f-9323-25eb08696e4d", + "metadata": {}, + "source": [ + "We can then take a look at the extracted spectrum in this box both in the cal file and the x1d file. **Information about this source!** \n", + "\n", + "*Note: The upturned edge effects seen in the 1-D spectrum are due to interpolation at the edges of the extraction box for the current flux calibration. This is also part of an ongoing calibration effort.*" + ] + }, { "cell_type": "code", "execution_count": null, @@ -767,7 +781,7 @@ "metadata": {}, "outputs": [], "source": [ - "# find the catalog for how many stars are between a specific magnitude \n", + "# find the catalog for how many sources are between a specific magnitude \n", "min_mag = 21.6\n", "max_mag = 21.7\n", "mag_cat = source_match_cat[(source_match_cat['isophotal_vegamag'] >= min_mag) & (source_match_cat['isophotal_vegamag'] <= max_mag)]\n", @@ -924,9 +938,7 @@ "\n", "### Final Visualization\n", "\n", - "Now that everything has been calibrated, it's useful to look at all of the extracted files for at least one of the sources. \n", - "\n", - "Continue to explore further, including using the [spec3 stage](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_spec3.html) of the pipeline!" + "Now that everything has been calibrated, it's useful to look at all of the extracted files. The cal and x1d files from spec2 are extracted at each dither step, which is shown below. Spec3 then turns the individual dither x1d files into a single combined spectrum per grism and filter for each source." ] }, { @@ -1093,6 +1105,14 @@ "fig.tight_layout()" ] }, + { + "cell_type": "markdown", + "id": "b115e0b7-459b-4688-a27e-b8a0afb22da3", + "metadata": {}, + "source": [ + "Continue to explore further, including using the [spec3 stage](https://jwst-pipeline.readthedocs.io/en/latest/jwst/pipeline/calwebb_spec3.html) of the pipeline!" + ] + }, { "cell_type": "markdown", "id": "d252a65a-1be8-4e0b-ae67-80cac8fc4770", From 1992884aac191a8275b2fe8d0f3540d40cd97007 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 9 Feb 2024 18:04:36 -0500 Subject: [PATCH 36/62] minor text updates --- .../NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index a32d80e16..4e772c320 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -960,8 +960,8 @@ "outputs": [], "source": [ "# with a known RA/Dec\n", - "desired_ra = 53.15437048946369 #53.16237 \n", - "desired_dec = -27.771689847051736 #-27.775\n", + "desired_ra = 53.15437048946369\n", + "desired_dec = -27.771689847051736\n", "\n", "c = SkyCoord(ra=desired_ra*u.degree, dec=desired_dec*u.degree)\n", "nearest_id, distance_2d, distance_3d = c.match_to_catalog_sky(cat['sky_centroid']) \n", @@ -995,6 +995,7 @@ "outputs": [], "source": [ "# alternatively with a known source number\n", + "# note that this number might be different for you depending on pipeline versions and parameters changed.\n", "source = 118\n", "\n", "wh_source = np.where(np.array(cat['label'] == source))[0][0]\n", From 641dd1a11d06694fc051aa368b1a0a4d5110a53e Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 15 Feb 2024 13:50:47 -0500 Subject: [PATCH 37/62] A few more helpful comments & removed not downloading all of the files as it will make the other notebooks more confusing to not have all of the files --- .../00_niriss_mast_query_data_setup.ipynb | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index 6f7f85b60..169bda651 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -150,7 +150,7 @@ " obs_id=['jw02079-o004*'], # Searching for PID 2079 observation 004\n", " ) \n", "\n", - "print(len(obs_id_table), 'files found')" + "print(len(obs_id_table), 'files found') # 611 files" ] }, { @@ -184,7 +184,7 @@ "print(\"How many batches?\", len(obs_batches))\n", "\n", "single_group = obs_batches[0] # Useful to inspect the files obtained from one group\n", - "print(\"Inspect the first batch to ensure that it matches expectations of what you want downloaded:\")\n", + "print(\"Inspect the first batch to ensure that it matches expectations of what you want downloaded:\") \n", "single_group['obs_collection', 'instrument_name', 'filters', 'target_name', 'obs_id', 's_ra', 's_dec', 't_exptime', 'proposal_id']" ] }, @@ -331,9 +331,7 @@ "cell_type": "code", "execution_count": null, "id": "616ff15c", - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "# Now let's get the products for each batch of observations, and filter down to only the products of interest.\n", @@ -344,7 +342,7 @@ " \n", " # Make a list of the `obsid` identifiers from our Astropy table of observations to get products for.\n", " obsids = batch['obsid']\n", - " print('Working with the following ``obsid``s:')\n", + " print('Working with the following obsids:')\n", " for number, obs_text in zip(obsids, batch['obs_id']):\n", " print(f\"{number} : {obs_text}\")\n", " \n", @@ -357,15 +355,14 @@ " productSubGroupDescription=[\"RATE\", \"ASN\"], # Not using \"UNCAL\" here since we can start with the rate files\n", " calib_level=[2, 3] # not using 1 here since not getting the UNCAL files\n", " )\n", + " \n", " # Download products for these records.\n", + " # The `flat=True` option stores all files in a single directory specified by `download_dir`.\n", " manifest = Observations.download_products(filtered_products,\n", " download_dir=download_dir,\n", " flat=True, # astroquery v0.4.7 or later only\n", " ) \n", - " print('Products downloaded:\\n', filtered_products['productFilename'])\n", - " \n", - " # only downloading the first batch of 5 observations\n", - " break # comment this out if you want to download everything, which you'll need to run the rest of the notebooks" + " print('Products downloaded:\\n', filtered_products['productFilename'])" ] }, { @@ -373,7 +370,7 @@ "id": "31ff8769-1240-452d-b84c-eca69d53a13f", "metadata": {}, "source": [ - "If continuing on with the WFSS notebooks, let's double check that we've downloaded all of the files that we need for the remaining notebooks. There should be 143 files downloaded. " + "If continuing on with the WFSS notebooks, let's double check that we've downloaded all of the files that we need for the remaining notebooks. There should be 149 files downloaded. " ] }, { @@ -446,7 +443,7 @@ "print('Saved:', outfile)\n", "\n", "# Look at the resulting dataframe\n", - "rate_df" + "rate_dfsort" ] }, { @@ -454,7 +451,7 @@ "id": "4f49059b-cc92-448f-aca1-b5af87b0c208", "metadata": {}, "source": [ - "In particular, let's look at the observation sequence these rate files follow. We have sorted the filenames above, so they should already be in time order in the dataframe. \n", + "In particular, let's look at the observation sequence these rate files follow. We have sorted the files above by exposure time, so they should already be in time order in the dataframe. \n", "\n", "`FILTER = CLEAR` indicates a direct image while `FILTER=GR150R` or `FILTER=GR150C` indicates a dispersed image. PUPIL is the blocking filter used. The first 14 exposures make up the first sequence set of direct image -> grism -> direct image -> grism. There are also multiple dither positions for the dispersed images and the direct images. The multiple direct image dithers will be combined in image3, while the multiple dispersed images can be combined in spec3. " ] @@ -490,7 +487,7 @@ "rows = int(np.ceil(14 / cols))\n", "\n", "# loop over the first 14 rate files and plot them\n", - "for plt_num, rf in enumerate(rate_df[0:14]['FILENAME']):\n", + "for plt_num, rf in enumerate(rate_dfsort[0:14]['FILENAME']):\n", "\n", " # determine where the subplot should be\n", " xpos = (plt_num%40) % cols\n", From c0ed3dea15bbdf3bef8a78421653dca09358e5fc Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 15 Feb 2024 15:53:33 -0500 Subject: [PATCH 38/62] Fixing the magnitude for the source we want to look at --- .../02_niriss_wfss_spec2.ipynb | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index ddc4ab31f..aed131cee 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -771,7 +771,7 @@ "\n", "# source 118 should have a Vega mag of ~21.68\n", "source_id = 118\n", - "print(f\"Magnitude for source in previous notebook: source {source_id} : {source_match_cat[source_match_cat['label'] == source_id]['isophotal_vegamag'][0]}\")" + "print(f\"Magnitude for source in previous notebook (source {source_id}) : {source_match_cat[source_match_cat['label'] == source_id]['isophotal_vegamag'][0]}\")" ] }, { @@ -781,9 +781,9 @@ "metadata": {}, "outputs": [], "source": [ - "# find the catalog for how many sources are between a specific magnitude \n", - "min_mag = 21.6\n", - "max_mag = 21.7\n", + "# find the catalog for how many sources are between a specific magnitude (21.18 to make sure to include our example source)\n", + "min_mag = 21.1\n", + "max_mag = 21.2\n", "mag_cat = source_match_cat[(source_match_cat['isophotal_vegamag'] >= min_mag) & (source_match_cat['isophotal_vegamag'] <= max_mag)]\n", "\n", "mag_cat['label', 'xcentroid', 'ycentroid', 'sky_centroid', 'is_extended', 'isophotal_abmag', 'isophotal_vegamag']" @@ -988,6 +988,16 @@ "##### Look at all of the sources for a single file" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "59b1458a-1544-4593-aea9-987764c0ac9a", + "metadata": {}, + "outputs": [], + "source": [ + "ls new_catalog_calibrated/*jw02079004002_11101_00002*" + ] + }, { "cell_type": "code", "execution_count": null, @@ -995,7 +1005,7 @@ "metadata": {}, "outputs": [], "source": [ - "x1d_file = x1ds[2]\n", + "x1d_file = os.path.join(source_outdir, 'jw02079004002_11101_00002_nis_x1d.fits') # this is a nice spectrum of 118 above\n", "cal_file = x1d_file.replace('x1d.fits', 'cal.fits')\n", "with fits.open(x1d_file) as x1d_hdu, fits.open(cal_file) as cal_hdu:\n", "\n", @@ -1036,8 +1046,7 @@ "## grism data\n", "\n", "# for the dispersed image plot\n", - "x1d_file = x1ds[0]\n", - "cal_file = x1d_file.replace('x1d.fits', 'cal.fits')\n", + "# x1d_file and cal_file use the same root as we are looking at for a single source\n", "rate_file = os.path.basename(x1d_file.replace('x1d.fits', 'rate.fits'))\n", "\n", "# fill in the nan values from the bad pixels with zero to make it easier to look at\n", From dfe45e7aa552a1bf97e03aac7ac64b01cdf22b35 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 15 Feb 2024 15:53:59 -0500 Subject: [PATCH 39/62] adding the number of files to expect after the print line --- .../NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb index 573fa1ba5..9fd67e87c 100644 --- a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb @@ -177,7 +177,7 @@ "# A quick sanity check to ensure that the files were calibrated.\n", "# if this is zero, check the log message above for any errors that may have occurred during the calibration\n", "wcsstep_files = glob.glob(os.path.join(data_dir_out, '*assignwcsstep*'))\n", - "print(len(wcsstep_files), 'assignwcsstep files written')" + "print(len(wcsstep_files), 'assignwcsstep files written') # 29 files should have been written" ] }, { @@ -215,7 +215,7 @@ "# A quick sanity check to ensure that the files were calibrated.\n", "# if this is zero, check the log message above for any errors that may have occurred during the calibration\n", "flatfield_files = glob.glob(os.path.join(data_dir_out, '*flatfieldstep*'))\n", - "print(len(flatfield_files), 'flatfieldstep files written')" + "print(len(flatfield_files), 'flatfieldstep files written') # 29 files should have been written (matching the wcs step)" ] }, { From 64e5f6c05084c96f3dab632331335fe631a07064 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 21 Mar 2024 16:57:39 -0400 Subject: [PATCH 40/62] changing imviz to use sky coordinates --- .../01_niriss_wfss_image2_image3.ipynb | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 4e772c320..96e7ff687 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -577,6 +577,13 @@ "imviz = Imviz()\n", "viewer = imviz.default_viewer\n", "\n", + "# this aligns the image to use the WCS coordinates\n", + "linking = imviz.plugins['Links Control']\n", + "linking.link_type = 'WCS'\n", + "\n", + "# plot each i2d image\n", + "catalogs = [] # for plotting the catalogs\n", + "labels = [] # for plotting the catalogs\n", "for img in image3_i2d:\n", " print(f'Plotting: {img}')\n", " label = f\"obs{fits.getval(img, 'OBSERVTN')} {fits.getval(img, 'PUPIL')}\"\n", @@ -584,17 +591,22 @@ " warnings.simplefilter('ignore')\n", " imviz.load_data(img, data_label=label)\n", "\n", - " # this aligns the image to use the WCS coordinates\n", - " linking = imviz.plugins['Links Control']\n", - " linking.link_type = 'WCS'\n", + " # save info to plot the catalogs next\n", + " catalogs.append(img.replace('i2d.fits', 'cat.ecsv'))\n", + " labels.append(label)\n", "\n", - " # also plot the associated catalog\n", - " cat = Table.read(img.replace('i2d.fits', 'cat.ecsv'))\n", + "# also plot the associated catalog\n", + "# this needs to be a separate loop due to linking in imviz when using sky coordinates\n", + "for catname, label in zip(catalogs, labels):\n", + " cat = Table.read(catname)\n", + " \n", " # format the table into the format imviz expects\n", - " t_xy = Table({'x': cat['xcentroid'],\n", - " 'y': cat['ycentroid']})\n", + " sky_coords = Table({'coord': [SkyCoord(ra=cat['sky_centroid'].ra.degree,\n", + " dec=cat['sky_centroid'].dec.degree,\n", + " unit=\"deg\")]})\n", + "\n", " viewer.marker = {'color': 'orange', 'alpha': 1, 'markersize': 20, 'fill': False}\n", - " viewer.add_markers(t_xy, marker_name=f\"{label} catalog\")\n", + " viewer.add_markers(sky_coords, use_skycoord=True, marker_name=f\"{label} catalog\")\n", "\n", "# This changes the stretch of all of the images\n", "plotopt = imviz.plugins['Plot Options']\n", From 966846b7de53ebd8eee0cde64f40d67ef0c7411c Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 21 Mar 2024 18:39:59 -0400 Subject: [PATCH 41/62] Adding info about last CRDS and pipeline version these notebooks were tested with --- .../01_niriss_wfss_image2_image3.ipynb | 18 +++++++++++------- .../02_niriss_wfss_spec2.ipynb | 12 ++++++++---- .../extra_niriss_individual_steps.ipynb | 13 +++++++++---- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 96e7ff687..953087769 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -11,7 +11,7 @@ "\n", "**Use case**: The default parameters for the pipeline do not extract the expected sources, so custom parameters need to be set to obtain new combined image and source catalog.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079 observation 004. This should be stored in a single directory `data`, and can be downloaded from the previous notebook, 00_niriss_mast_query_data_setup.ipynb.
\n", - "**Tools**: astropy, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, warnings
\n", + "**Tools**: astropy, crds, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, warnings
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -28,7 +28,9 @@ " - [Manually Editing the Source Catalog](#manual_cat)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", - "**First Published**: January 2024" + "**First Published**: March 2024\n", + "\n", + "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1215.pmap." ] }, { @@ -83,7 +85,6 @@ "from matplotlib import pyplot as plt\n", "%matplotlib inline\n", "\n", - "import jwst\n", "from jwst.pipeline import Image2Pipeline\n", "from jwst.pipeline import Image3Pipeline" ] @@ -93,7 +94,7 @@ "id": "fb035a4d-e63c-4125-b083-462e3353e49f", "metadata": {}, "source": [ - "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available, check here" + "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available or install a previous version, check [GitHub](https://github.com/spacetelescope/jwst#software-vs-dms-build-version-map\"). Also verify what [CRDS version](https://jwst-crds.stsci.edu/) you are using. [CRDS documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/user_documentation/reference_files_crds.html) explains how to set a specific context to use in the JWST pipeline. If either of these values are different from the last tested note above there may be differences in this notebook." ] }, { @@ -103,7 +104,10 @@ "metadata": {}, "outputs": [], "source": [ - "print('jwst:', jwst.__version__)" + "import jwst\n", + "import crds\n", + "print('JWST Pipeliene Version:', jwst.__version__)\n", + "print('CRDS Context:', crds.get_context_name('jwst'))" ] }, { @@ -289,8 +293,8 @@ " asn_data = json.load(open(img2_asn))\n", " cal_file = f\"{asn_data['products'][0]['name']}_cal.fits\"\n", " if os.path.exists(cal_file):\n", - " print(cal_file, 'cal file already exists.')\n", - " continue\n", + " print(cal_file, 'cal file already exists.')\n", + " continue\n", " # if not, calibrated with image2\n", " img2 = Image2Pipeline.call(img2_asn, save_results=True)" ] diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index aed131cee..85965dc61 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -11,7 +11,7 @@ "\n", "**Use case**: After creating a custom source catalog, spec2 should be run on the dispersed WFSS images.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079 obs 004. This should be stored in a single directory `data`, and can be downloaded from the notebook 00_niriss_mast_query_data_setup.ipynb.
\n", - "**Tools**: astropy, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, shutil
\n", + "**Tools**: astropy, crds, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, shutil
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -26,7 +26,9 @@ " - [Final Visualization](#final_visualize)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", - "**First Published**: February 2024" + "**First Published**: March 2024\n", + "\n", + "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1215.pmap." ] }, { @@ -89,7 +91,7 @@ "id": "64b6e8d7-64a0-4a3b-9776-29b181452ff9", "metadata": {}, "source": [ - "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available, check here" + "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available or install a previous version, check [GitHub](https://github.com/spacetelescope/jwst#software-vs-dms-build-version-map\"). Also verify what [CRDS version](https://jwst-crds.stsci.edu/) you are using. [CRDS documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/user_documentation/reference_files_crds.html) explains how to set a specific context to use in the JWST pipeline. If either of these values are different from the last tested note above there may be differences in this notebook." ] }, { @@ -100,7 +102,9 @@ "outputs": [], "source": [ "import jwst\n", - "print('jwst',jwst.__version__)" + "import crds\n", + "print('JWST Pipeliene Version:', jwst.__version__)\n", + "print('CRDS Context:', crds.get_context_name('jwst'))" ] }, { diff --git a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb index 9fd67e87c..c016d2bab 100644 --- a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb @@ -11,7 +11,7 @@ "\n", "**Use case**: When using a package outside of the standard JWST pipeline, there may be certain steps that are still helpful to utilize within the JWST pipeline.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: astropy, glob, jdaviz, jwst, matplotlib, numpy, os, pandas, warnings
\n", + "**Tools**: astropy, crds, glob, jdaviz, jwst, matplotlib, numpy, os, pandas, warnings
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -22,7 +22,9 @@ " - [Compare Rate vs. Flat Fielded Data](#compare)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu)
\n", - "**Last modified**: February 2024" + "**Last modified**: March 2024\n", + "\n", + "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1215.pmap." ] }, { @@ -76,7 +78,7 @@ "id": "3c208323-56f4-4fb5-b8c0-750b65b525aa", "metadata": {}, "source": [ - "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available, check here" + "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available or install a previous version, check [GitHub](https://github.com/spacetelescope/jwst#software-vs-dms-build-version-map\"). Also verify what [CRDS version](https://jwst-crds.stsci.edu/) you are using. [CRDS documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/user_documentation/reference_files_crds.html) explains how to set a specific context to use in the JWST pipeline. If either of these values are different from the last tested note above there may be differences in this notebook." ] }, { @@ -86,7 +88,10 @@ "metadata": {}, "outputs": [], "source": [ - "print('jwst:', jwst.__version__)" + "import jwst\n", + "import crds\n", + "print('JWST Pipeliene Version:', jwst.__version__)\n", + "print('CRDS Context:', crds.get_context_name('jwst'))" ] }, { From 2b09f5e08c28f977e55b6287e8f2530d9aeebdb2 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 21 Mar 2024 22:02:56 -0400 Subject: [PATCH 42/62] Adding box link files to run notebooks independently --- .../01_niriss_wfss_image2_image3.ipynb | 34 ++++++++- .../02_niriss_wfss_spec2.ipynb | 73 ++++++++++++++----- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 953087769..5674bbb62 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -11,7 +11,7 @@ "\n", "**Use case**: The default parameters for the pipeline do not extract the expected sources, so custom parameters need to be set to obtain new combined image and source catalog.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079 observation 004. This should be stored in a single directory `data`, and can be downloaded from the previous notebook, 00_niriss_mast_query_data_setup.ipynb.
\n", - "**Tools**: astropy, crds, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, warnings
\n", + "**Tools**: astropy, crds, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, urllib, warnings, zipfile
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -73,6 +73,8 @@ "import glob\n", "import json\n", "import warnings\n", + "import urllib\n", + "import zipfile\n", "import numpy as np\n", "import pandas as pd\n", "\n", @@ -138,6 +140,36 @@ "The association files expect that 1) all of the data are in the same directory and 2) that you are performing the pipeline call also in that directory. Because of that, we need to change into the data directory to run the imaging pipelines." ] }, + { + "cell_type": "raw", + "id": "e34b0097-47df-468e-ad12-a7ca1d5a9565", + "metadata": {}, + "source": [ + "# if you have not downloaded the data from notebook 00, run this cell in \"code\" mode rather than \"raw\"\n", + "\n", + "# Download uncalibrated data from Box into the data directory:\n", + "boxlink = 'https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/niriss_wfss_advanced/niriss_wfss_advanced_01_input.zip'\n", + "boxfile = os.path.basename(boxlink)\n", + "urllib.request.urlretrieve(boxlink, boxfile)\n", + "\n", + "zf = zipfile.ZipFile(boxfile, 'r')\n", + "zf.extractall(path=data_dir)\n", + "\n", + "# move the files downloaded from the box file into the top level data directory\n", + "box_download_dir = os.path.join(data_dir, boxfile.split('.zip')[0])\n", + "for filename in glob.glob(os.path.join(box_download_dir, '*')):\n", + " if '.csv' in filename:\n", + " # move to the current directory\n", + " os.rename(filename, os.path.basename(filename))\n", + " else:\n", + " # move to the data directory \n", + " os.rename(filename, os.path.join(data_dir, os.path.basename(filename)))\n", + "\n", + "# remove unnecessary files now\n", + "os.remove(boxfile)\n", + "os.rmdir(box_download_dir)" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index 85965dc61..f3e052f1e 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -11,7 +11,7 @@ "\n", "**Use case**: After creating a custom source catalog, spec2 should be run on the dispersed WFSS images.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079 obs 004. This should be stored in a single directory `data`, and can be downloaded from the notebook 00_niriss_mast_query_data_setup.ipynb.
\n", - "**Tools**: astropy, crds, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, shutil
\n", + "**Tools**: astropy, crds, glob, jdaviz, json, jwst, matplotlib, numpy, os, pandas, shutil, urllib, zipfile
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -71,6 +71,8 @@ "import json\n", "import os\n", "import shutil\n", + "import urllib\n", + "import zipfile\n", "import numpy as np\n", "import pandas as pd\n", "\n", @@ -137,12 +139,53 @@ "outputs": [], "source": [ "data_dir = 'data'\n", - "custom_run_spec2 = 'custom_spec2'\n", - "custom_run_image3 = os.path.join(data_dir, 'custom_image3_calibrated') # results of custom image3 calibration\n", + "if not os.path.exists(data_dir):\n", + " os.mkdir(data_dir)\n", + " \n", + "custom_run_spec2 = 'custom_spec2' # saving files here in this notebook\n", + "custom_run_image3 = 'custom_image3_calibrated' # results of custom image3 calibration\n", + "\n", + "# if the directories dont't exist yet, make it\n", + "for custom_dir in [custom_run_spec2, custom_run_image3]:\n", + " if not os.path.exists(os.path.join(data_dir, custom_dir)):\n", + " os.mkdir(os.path.join(data_dir, custom_dir))" + ] + }, + { + "cell_type": "raw", + "id": "e3611a58-e99e-47aa-b190-b821eb60e47f", + "metadata": {}, + "source": [ + "# if you have not downloaded the data from notebook 00 or have not run notebook 01, run this cell in \"code\" mode rather than \"raw\"\n", + "\n", + "# Download uncalibrated data from Box into the data directory:\n", + "boxlink = 'https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/niriss_wfss_advanced/niriss_wfss_advanced_02_input.zip'\n", + "boxfile = os.path.basename(boxlink)\n", + "urllib.request.urlretrieve(boxlink, boxfile)\n", + "\n", + "zf = zipfile.ZipFile(boxfile, 'r')\n", + "zf.extractall(path=data_dir)\n", + "\n", + "# move the files downloaded from the box file into the top level data directory\n", + "box_download_dir = os.path.join(data_dir, boxfile.split('.zip')[0])\n", "\n", - "# if the custom spec2 run directory doesn't exist yet, make it\n", - "if not os.path.exists(os.path.join(data_dir, custom_run_spec2)):\n", - " os.mkdir(os.path.join(data_dir, custom_run_spec2))" + "for filename in glob.glob(os.path.join(box_download_dir, '*')):\n", + " if '.csv' in filename:\n", + " # move to the current directory\n", + " os.rename(filename, os.path.basename(filename))\n", + " elif '_segm.fits' in filename or '_cat.ecsv' in filename:\n", + " # move the image2 products to the appropriate directory\n", + " os.rename(filename, os.path.join(data_dir, custom_run_spec2, os.path.basename(filename)))\n", + " elif '_i2d.fits' in filename:\n", + " # move image3 products to their directory, too\n", + " os.rename(filename, os.path.join(data_dir, custom_run_image3, os.path.basename(filename)))\n", + " else:\n", + " # move to the data directory \n", + " os.rename(filename, os.path.join(data_dir, os.path.basename(filename)))\n", + " \n", + "# remove unnecessary files now\n", + "os.remove(boxfile)\n", + "os.rmdir(box_download_dir)" ] }, { @@ -198,8 +241,8 @@ "outputs": [], "source": [ "# copy all of the necessary image3 output files\n", - "cats = glob.glob(os.path.join(custom_run_image3, '*source*_cat.ecsv')) # copy both the source-match and source118 catalogs\n", - "segm = glob.glob(os.path.join(custom_run_image3, '*_segm.fits'))\n", + "cats = glob.glob(os.path.join(data_dir, custom_run_image3, '*source*_cat.ecsv')) # copy both the source-match and source118 catalogs\n", + "segm = glob.glob(os.path.join(data_dir, custom_run_image3, '*_segm.fits'))\n", "\n", "for image3_file in cats + segm:\n", " if os.path.exists(image3_file):\n", @@ -912,7 +955,7 @@ "id": "c5dd5d4c-5ef0-4690-a4da-a795c295a313", "metadata": {}, "source": [ - "Now when we calibrate everything, it should take a lot less time because there are a limited number of sources." + "Now when we calibrate everything, for a single file it should take a lot less time because there are a limited number of sources. However, we will calibrate all of the files in this visit, so this cell might take a bit of time to run." ] }, { @@ -992,16 +1035,6 @@ "##### Look at all of the sources for a single file" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "59b1458a-1544-4593-aea9-987764c0ac9a", - "metadata": {}, - "outputs": [], - "source": [ - "ls new_catalog_calibrated/*jw02079004002_11101_00002*" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1095,7 +1128,7 @@ "asn_data = json.load(open(fits.getval(x1d_file, 'ASNTABLE')))\n", "i2d_name = asn_data['products'][0]['members'][1]['expname']\n", "cat_name = asn_data['products'][0]['members'][2]['expname']\n", - "with fits.open(os.path.join('../../', custom_run_image3, i2d_name)) as i2d:\n", + "with fits.open(os.path.join('../../', data_dir, custom_run_image3, i2d_name)) as i2d:\n", " ax2.imshow(i2d[1].data, origin='lower', aspect='auto', vmin=0, vmax=np.nanmax(i2d[1].data)*0.01)\n", " ax2.set_title(f\"{os.path.basename(i2d_name).split('_i2d')[0]}\")\n", "\n", From cf9c08da5175d13ac250048e884a6291ec9dfa69 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 09:57:27 -0400 Subject: [PATCH 43/62] Initial commit of the requirements file --- notebooks/NIRISS_WFSS_advanced/requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 notebooks/NIRISS_WFSS_advanced/requirements.txt diff --git a/notebooks/NIRISS_WFSS_advanced/requirements.txt b/notebooks/NIRISS_WFSS_advanced/requirements.txt new file mode 100644 index 000000000..7ac4eefc3 --- /dev/null +++ b/notebooks/NIRISS_WFSS_advanced/requirements.txt @@ -0,0 +1,15 @@ +astropy >= 4.3.1 +astroquery >= 0.4.7 +crds +glob +jdaviz +json +jwst >= 1.12.5 +matplotlib +numpy +os +pandas +shutil +warnings +urllib +zipfile From 0650a6dab748066eaad23449c2614e4d7e6a8af5 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 10:59:32 -0400 Subject: [PATCH 44/62] Fixing most pep8 style issues --- .../00_niriss_mast_query_data_setup.ipynb | 52 ++++++------- .../01_niriss_wfss_image2_image3.ipynb | 78 ++++++++++--------- .../02_niriss_wfss_spec2.ipynb | 43 +++++----- .../extra_niriss_individual_steps.ipynb | 10 +-- 4 files changed, 89 insertions(+), 94 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index 169bda651..f3bbae269 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -46,8 +46,6 @@ "metadata": {}, "outputs": [], "source": [ - "from astropy import units as u\n", - "from astropy.coordinates import SkyCoord\n", "from astropy.io import fits\n", "from astroquery.mast import Observations\n", "from matplotlib import pyplot as plt\n", @@ -230,18 +228,18 @@ "outputs": [], "source": [ "# creating a dictionary of the above information to use for inspection of the filtering function\n", - "file_dict = {'uncal' : {'product_type' : 'SCIENCE',\n", - " 'productSubGroupDescription' : 'UNCAL',\n", - " 'calib_level' : [1]},\n", - " 'rate' : {'product_type' : 'SCIENCE',\n", - " 'productSubGroupDescription' : 'RATE',\n", - " 'calib_level' : [2]},\n", - " 'level2_association' : {'product_type' : 'INFO',\n", - " 'productSubGroupDescription' : 'ASN',\n", - " 'calib_level' : [2]},\n", - " 'level3_association' : {'product_type' : 'INFO',\n", - " 'productSubGroupDescription' : 'ASN',\n", - " 'calib_level' : [3]},\n", + "file_dict = {'uncal': {'product_type': 'SCIENCE',\n", + " 'productSubGroupDescription': 'UNCAL',\n", + " 'calib_level': [1]},\n", + " 'rate': {'product_type': 'SCIENCE',\n", + " 'productSubGroupDescription': 'RATE',\n", + " 'calib_level': [2]},\n", + " 'level2_association': {'product_type': 'INFO',\n", + " 'productSubGroupDescription': 'ASN',\n", + " 'calib_level': [2]},\n", + " 'level3_association': {'product_type': 'INFO',\n", + " 'productSubGroupDescription': 'ASN',\n", + " 'calib_level': [3]},\n", " }" ] }, @@ -254,7 +252,7 @@ }, "outputs": [], "source": [ - "## Look at the files existing for each of these different levels\n", + "# Look at the files existing for each of these different levels\n", "files_to_download = []\n", "for index, batch_exposure in enumerate(single_group):\n", " \n", @@ -412,16 +410,16 @@ " rate_hdr = fits.getheader(ratefile) # Primary header for each rate file\n", "\n", " # information we want to store that might be useful to us later for evaluating the data\n", - " temp_hdr_dict = {\"FILENAME\" : ratefile,\n", - " \"TARG_RA\" : [rate_hdr[\"TARG_RA\"]],\n", - " \"TARG_DEC\" : [rate_hdr[\"TARG_DEC\"]],\n", - " \"FILTER\" : [rate_hdr[\"FILTER\"]], # Grism; GR150R/GR150C\n", - " \"PUPIL\" : [rate_hdr[\"PUPIL\"]], # Filter used; F090W, F115W, F140M, F150W F158M, F200W\n", - " \"EXPSTART\" : [rate_hdr['EXPSTART']], # Exposure start time (MJD)\n", - " \"PATT_NUM\" : [rate_hdr[\"PATT_NUM\"]], # Position number within dither pattern for WFSS\n", - " \"NUMDTHPT\" : [rate_hdr[\"NUMDTHPT\"]], # Total number of points in entire dither pattern\n", - " \"XOFFSET\" : [rate_hdr[\"XOFFSET\"]], # X offset from pattern starting position for NIRISS (arcsec)\n", - " \"YOFFSET\" : [rate_hdr[\"YOFFSET\"]] # Y offset from pattern starting position for NIRISS (arcsec)\n", + " temp_hdr_dict = {\"FILENAME\": ratefile,\n", + " \"TARG_RA\": [rate_hdr[\"TARG_RA\"]],\n", + " \"TARG_DEC\": [rate_hdr[\"TARG_DEC\"]],\n", + " \"FILTER\": [rate_hdr[\"FILTER\"]], # Grism; GR150R/GR150C\n", + " \"PUPIL\": [rate_hdr[\"PUPIL\"]], # Filter used; F090W, F115W, F140M, F150W F158M, F200W\n", + " \"EXPSTART\": [rate_hdr['EXPSTART']], # Exposure start time (MJD)\n", + " \"PATT_NUM\": [rate_hdr[\"PATT_NUM\"]], # Position number within dither pattern for WFSS\n", + " \"NUMDTHPT\": [rate_hdr[\"NUMDTHPT\"]], # Total number of points in entire dither pattern\n", + " \"XOFFSET\": [rate_hdr[\"XOFFSET\"]], # X offset from pattern starting position for NIRISS (arcsec)\n", + " \"YOFFSET\": [rate_hdr[\"YOFFSET\"]], # Y offset from pattern starting position for NIRISS (arcsec)\n", " }\n", "\n", " # Turn the dictionary into a pandas dataframe\n", @@ -490,8 +488,8 @@ "for plt_num, rf in enumerate(rate_dfsort[0:14]['FILENAME']):\n", "\n", " # determine where the subplot should be\n", - " xpos = (plt_num%40) % cols\n", - " ypos = ((plt_num%40) // cols) # // to make it an int.\n", + " xpos = (plt_num % 40) % cols\n", + " ypos = ((plt_num % 40) // cols) # // to make it an int.\n", "\n", " # make the subplot\n", " ax = plt.subplot2grid((rows, cols), (ypos, xpos))\n", diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 5674bbb62..c013ab236 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -141,11 +141,13 @@ ] }, { - "cell_type": "raw", - "id": "e34b0097-47df-468e-ad12-a7ca1d5a9565", + "cell_type": "code", + "execution_count": null, + "id": "39fbaa8a-5062-4a2d-b8a9-f3764086aa91", "metadata": {}, + "outputs": [], "source": [ - "# if you have not downloaded the data from notebook 00, run this cell in \"code\" mode rather than \"raw\"\n", + "# if you have not downloaded the data from notebook 00, run this cell. Otherwise, feel free to skip it.\n", "\n", "# Download uncalibrated data from Box into the data directory:\n", "boxlink = 'https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/niriss_wfss_advanced/niriss_wfss_advanced_01_input.zip'\n", @@ -263,7 +265,7 @@ "metadata": {}, "outputs": [], "source": [ - "## look at one of the association files\n", + "# look at one of the association files\n", "asn_data = json.load(open(image2_asns[0]))\n", "for key, data in asn_data.items():\n", " print(f\"{key} : {data}\")" @@ -288,7 +290,7 @@ "metadata": {}, "outputs": [], "source": [ - "## in particular, take a closer look at the product filenames with the association file:\n", + "# in particular, take a closer look at the product filenames with the association file:\n", "for product in asn_data['products']:\n", " for key, value in product.items():\n", " if key == 'members':\n", @@ -325,8 +327,8 @@ " asn_data = json.load(open(img2_asn))\n", " cal_file = f\"{asn_data['products'][0]['name']}_cal.fits\"\n", " if os.path.exists(cal_file):\n", - " print(cal_file, 'cal file already exists.')\n", - " continue\n", + " print(cal_file, 'cal file already exists.')\n", + " continue\n", " # if not, calibrated with image2\n", " img2 = Image2Pipeline.call(img2_asn, save_results=True)" ] @@ -371,7 +373,7 @@ "metadata": {}, "outputs": [], "source": [ - "## look at one of the association files\n", + "# look at one of the association files\n", "image3_asn_data = json.load(open(image3_asns[0]))\n", "for key, data in image3_asn_data.items():\n", " print(f\"{key} : {data}\")" @@ -384,7 +386,7 @@ "metadata": {}, "outputs": [], "source": [ - "## in particular, take a closer look at the product filenames with the association file:\n", + "# in particular, take a closer look at the product filenames with the association file:\n", "for product in image3_asn_data['products']:\n", " for key, value in product.items():\n", " if key == 'members':\n", @@ -485,8 +487,8 @@ "for plt_num, img in enumerate(image3_i2d):\n", "\n", " # determine where the subplot should be\n", - " xpos = (plt_num%40) % cols\n", - " ypos = ((plt_num%40) // cols) # // to make it an int.\n", + " xpos = (plt_num % 40) % cols\n", + " ypos = ((plt_num % 40) // cols) # // to make it an int.\n", "\n", " # make the subplot\n", " ax = plt.subplot2grid((rows, cols), (ypos, xpos))\n", @@ -498,8 +500,8 @@ "\n", " # also plot the associated catalog\n", " cat = Table.read(img.replace('i2d.fits', 'cat.ecsv'))\n", - " extended_sources = cat[cat['is_extended'] == True]\n", - " point_sources = cat[cat['is_extended'] == False]\n", + " extended_sources = cat[cat['is_extended'] == 1] # 1 is True; i.e. is extended\n", + " point_sources = cat[cat['is_extended'] == 0] # 0 is False; i.e. is a point source\n", " ax.scatter(extended_sources['xcentroid'], extended_sources['ycentroid'], s=20, facecolors='None', edgecolors='orangered', alpha=0.9)\n", " ax.scatter(point_sources['xcentroid'], point_sources['ycentroid'], s=20, facecolors='None', edgecolors='dimgrey', alpha=0.9)\n", "\n", @@ -532,13 +534,13 @@ " cols = 2\n", " rows = len(i2d_images)\n", "\n", - " fig = plt.figure(figsize=(10, 10*(rows/2)) )\n", + " fig = plt.figure(figsize=(10, 10*(rows/2)))\n", "\n", " for plt_num, img in enumerate(np.sort(np.concatenate([segm_images, i2d_images]))):\n", " \n", " # determine where the subplot should be\n", - " xpos = (plt_num%40) % cols\n", - " ypos = ((plt_num%40) // cols) # // to make it an int.\n", + " xpos = (plt_num % 40) % cols\n", + " ypos = ((plt_num % 40) // cols) # // to make it an int.\n", "\n", " # make the subplot\n", " ax = plt.subplot2grid((rows, cols), (ypos, xpos))\n", @@ -555,8 +557,8 @@ " title = f\"obs{hdu[0].header['OBSERVTN']} {hdu[0].header['PUPIL']}\"\n", " \n", " # also plot the associated catalog\n", - " extended_sources = cat[cat['is_extended'] == True]\n", - " point_sources = cat[cat['is_extended'] == False]\n", + " extended_sources = cat[cat['is_extended'] == 1] # 1 is True; i.e. is extended\n", + " point_sources = cat[cat['is_extended'] == 0] # 0 is False; i.e. is a point source\n", " \n", " for color, sources in zip(['darkred', 'black'], [extended_sources, point_sources]):\n", " # plotting the sources\n", @@ -704,17 +706,17 @@ " # call the image3 pipeline in the same way as before, but add a few new modifications\n", " cust_img3 = Image3Pipeline.call(test_asn,\n", " steps={\n", - " 'source_catalog':{'kernel_fwhm':5.0,\n", - " 'snr_threshold':10.0,\n", - " 'npixels':50,\n", - " 'deblend':True,\n", + " 'source_catalog': {'kernel_fwhm': 5.0,\n", + " 'snr_threshold': 10.0,\n", + " 'npixels': 50,\n", + " 'deblend': True,\n", " },\n", - " 'tweakreg':{'snr_threshold':20,\n", - " 'abs_refcat':'GAIADR3',\n", - " 'save_catalogs':True,\n", - " 'searchrad':3.0,\n", - " 'kernel_fwhm':2.302,\n", - " 'fitgeometry':'shift',\n", + " 'tweakreg': {'snr_threshold': 20,\n", + " 'abs_refcat': 'GAIADR3',\n", + " 'save_catalogs': True,\n", + " 'searchrad': 3.0,\n", + " 'kernel_fwhm': 2.302,\n", + " 'fitgeometry': 'shift',\n", " },\n", " },\n", " save_results=True,\n", @@ -808,17 +810,17 @@ " # call the image3 pipeline in the same way as before, but add a few new modifications\n", " cust_img3 = Image3Pipeline.call(img3_asn,\n", " steps={\n", - " 'source_catalog':{'kernel_fwhm':5.0,\n", - " 'snr_threshold':10.0,\n", - " 'npixels':50,\n", - " 'deblend':True,\n", + " 'source_catalog': {'kernel_fwhm': 5.0,\n", + " 'snr_threshold': 10.0,\n", + " 'npixels': 50,\n", + " 'deblend': True,\n", " },\n", - " 'tweakreg':{'snr_threshold':20,\n", - " 'abs_refcat':'GAIADR3',\n", - " 'save_catalogs':True,\n", - " 'searchrad':3.0,\n", - " 'kernel_fwhm':2.302,\n", - " 'fitgeometry':'shift',\n", + " 'tweakreg': {'snr_threshold': 20,\n", + " 'abs_refcat': 'GAIADR3',\n", + " 'save_catalogs': True,\n", + " 'searchrad': 3.0,\n", + " 'kernel_fwhm': 2.302,\n", + " 'fitgeometry': 'shift',\n", " },\n", " },\n", " save_results=True,\n", diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index f3e052f1e..e21575552 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -79,13 +79,11 @@ "from astropy.table import Table\n", "from astropy.io import fits\n", "\n", - "from jdaviz import Mosviz\n", "from matplotlib import pyplot as plt\n", "from matplotlib import patches\n", "%matplotlib inline\n", "\n", - "from jwst.pipeline import Spec2Pipeline\n", - "from jwst import datamodels" + "from jwst.pipeline import Spec2Pipeline" ] }, { @@ -152,11 +150,13 @@ ] }, { - "cell_type": "raw", - "id": "e3611a58-e99e-47aa-b190-b821eb60e47f", + "cell_type": "code", + "execution_count": null, + "id": "634e2f5f-8a49-4eea-83f6-850456436635", "metadata": {}, + "outputs": [], "source": [ - "# if you have not downloaded the data from notebook 00 or have not run notebook 01, run this cell in \"code\" mode rather than \"raw\"\n", + "# if you have not downloaded the data from notebook 00 or have not run notebook 01, run this cell. Otherwise, feel free to skip it.\n", "\n", "# Download uncalibrated data from Box into the data directory:\n", "boxlink = 'https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/niriss_wfss_advanced/niriss_wfss_advanced_02_input.zip'\n", @@ -343,7 +343,7 @@ "metadata": {}, "outputs": [], "source": [ - "## look at one of the association files\n", + "# look at one of the association files\n", "asn_data = json.load(open(spec2_asns[0]))\n", "for key, data in asn_data.items():\n", " print(f\"{key} : {data}\")" @@ -356,7 +356,7 @@ "metadata": {}, "outputs": [], "source": [ - "## in particular, take a closer look at the product filenames with the association file:\n", + "# in particular, take a closer look at the product filenames with the association file:\n", "for product in asn_data['products']:\n", " for key, value in product.items():\n", " if key == 'members':\n", @@ -414,7 +414,7 @@ "metadata": {}, "outputs": [], "source": [ - "## double check that things still look ok:\n", + "# double check that things still look ok:\n", "asn_check = json.load(open(spec2_asns[0]))\n", "for product in asn_check['products']:\n", " for key, value in product.items():\n", @@ -611,7 +611,8 @@ " \n", " # look for cal extension, too, but only in the SCI extension; \n", " # fill in with a source ID of -999 for all other extensions to get the right extension value\n", - " cal_source_ids = np.array([cal_hdu[ext].header['SOURCEID'] if cal_hdu[ext].header['EXTNAME'] == 'SCI' else -999 for ext in range(len(cal_hdu))[1:-1] ]) \n", + " cal_source_ids = np.array([cal_hdu[ext].header['SOURCEID'] if cal_hdu[ext].header['EXTNAME'] == 'SCI' \\\n", + " else -999 for ext in range(len(cal_hdu))[1:-1]]) \n", " wh_cal = np.where(cal_source_ids == source_id)[0][0] + 1 # need to add 1 for the primary header\n", "\n", " if info:\n", @@ -773,7 +774,7 @@ " print(x1d_file, ': x1d file already exists.')\n", "else:\n", " spec2 = Spec2Pipeline.call(spec2_asns[0],\n", - " steps={'extract_2d':{'wfss_nbright' : 10,},},\n", + " steps={'extract_2d': {'wfss_nbright': 10,}, },\n", " save_results=True,\n", " output_dir=param_outdir)" ] @@ -818,7 +819,8 @@ "\n", "# source 118 should have a Vega mag of ~21.68\n", "source_id = 118\n", - "print(f\"Magnitude for source in previous notebook (source {source_id}) : {source_match_cat[source_match_cat['label'] == source_id]['isophotal_vegamag'][0]}\")" + "source_mag = source_match_cat[source_match_cat['label'] == source_id]['isophotal_vegamag'][0]\n", + "print(f\"Magnitude for source in previous notebook (source {source_id}) : {source_mag}\")" ] }, { @@ -843,7 +845,7 @@ "metadata": {}, "outputs": [], "source": [ - "mag_source_ext = f'mag-limit_cat.ecsv'\n", + "mag_source_ext = 'mag-limit_cat.ecsv'\n", "\n", "new_cat = Table(mag_cat) # turn the row instance into a dataframe again\n", "\n", @@ -936,13 +938,12 @@ }, "outputs": [], "source": [ - "## double check that the source catalog has been changed\n", + "# double check that the source catalog has been changed\n", "for spec2_asn in spec2_asns:\n", " asn_check = json.load(open(spec2_asn))\n", " for product in asn_check['products']:\n", " for key, value in product.items():\n", " if key == 'members':\n", - " #print(f\"{key}:\")\n", " for member in value:\n", " if member['exptype'] == 'sourcecat':\n", " print(f\" {member['exptype']}: {member['expname']}\")\n", @@ -965,7 +966,7 @@ "metadata": {}, "outputs": [], "source": [ - "## calibrate with the new source catalog\n", + "# calibrate with the new source catalog\n", "for spec2_asn in spec2_asns:\n", " # check if the calibrated file already exists\n", " asn_data = json.load(open(spec2_asn))\n", @@ -1003,7 +1004,7 @@ "metadata": {}, "outputs": [], "source": [ - "#### Explore the new Data\n", + "# Explore the new Data\n", "x1ds = np.sort(glob.glob(os.path.join(source_outdir, \"*x1d.fits\")))\n", "\n", "# get a list of all of the source IDs from the first file to look at for this example\n", @@ -1080,7 +1081,7 @@ "# setting up the figure\n", "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 7))\n", "\n", - "## grism data\n", + "# **grism data\n", "\n", "# for the dispersed image plot\n", "# x1d_file and cal_file use the same root as we are looking at for a single source\n", @@ -1124,7 +1125,7 @@ " \n", " ax1.set_title(f\"{os.path.basename(x1d_file).split('_nis')[0]}: {cal_hdu[0].header['FILTER']} {cal_hdu[0].header['PUPIL']}\")\n", "\n", - "## imaging data\n", + "# **imaging data\n", "asn_data = json.load(open(fits.getval(x1d_file, 'ASNTABLE')))\n", "i2d_name = asn_data['products'][0]['members'][1]['expname']\n", "cat_name = asn_data['products'][0]['members'][2]['expname']\n", @@ -1134,8 +1135,8 @@ "\n", "# also plot the associated catalog\n", "cat = Table.read(cat_name)\n", - "extended_sources = cat[cat['is_extended'] == True]\n", - "point_sources = cat[cat['is_extended'] == False]\n", + "extended_sources = cat[cat['is_extended'] == 1] # 1 is True; i.e. is extended\n", + "point_sources = cat[cat['is_extended'] == 0] # 0 is False; i.e. is a point source\n", "\n", "for color, sources in zip(['darkred', 'black'], [extended_sources, point_sources]):\n", " # plotting the sources\n", diff --git a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb index c016d2bab..c1974886e 100644 --- a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb @@ -11,7 +11,7 @@ "\n", "**Use case**: When using a package outside of the standard JWST pipeline, there may be certain steps that are still helpful to utilize within the JWST pipeline.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: astropy, crds, glob, jdaviz, jwst, matplotlib, numpy, os, pandas, warnings
\n", + "**Tools**: astropy, crds, glob, jwst, matplotlib, numpy, os, pandas
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -45,20 +45,14 @@ "source": [ "import os\n", "import glob\n", - "import warnings\n", "import numpy as np\n", "import pandas as pd\n", "\n", - "import astropy.units as u\n", - "from astropy.io import ascii, fits\n", - "from astropy.table import Table\n", - "from astropy.coordinates import SkyCoord, Angle\n", + "from astropy.io import fits\n", "\n", - "from jdaviz import Imviz\n", "from matplotlib import pyplot as plt\n", "%matplotlib inline\n", "\n", - "import jwst\n", "from jwst.assign_wcs import AssignWcsStep\n", "from jwst.flatfield import FlatFieldStep" ] From 5e527b11c03067d708921cf298ceb47d464e23c8 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 11:19:09 -0400 Subject: [PATCH 45/62] glob -> glob2 --- notebooks/NIRISS_WFSS_advanced/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/NIRISS_WFSS_advanced/requirements.txt b/notebooks/NIRISS_WFSS_advanced/requirements.txt index 7ac4eefc3..5b0e20267 100644 --- a/notebooks/NIRISS_WFSS_advanced/requirements.txt +++ b/notebooks/NIRISS_WFSS_advanced/requirements.txt @@ -1,7 +1,7 @@ astropy >= 4.3.1 astroquery >= 0.4.7 crds -glob +glob2 jdaviz json jwst >= 1.12.5 From 0899da41c6bf40f3f74987f98a5828902358e14c Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 11:22:15 -0400 Subject: [PATCH 46/62] Removing built in modules --- notebooks/NIRISS_WFSS_advanced/requirements.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/requirements.txt b/notebooks/NIRISS_WFSS_advanced/requirements.txt index 5b0e20267..ebed5dd67 100644 --- a/notebooks/NIRISS_WFSS_advanced/requirements.txt +++ b/notebooks/NIRISS_WFSS_advanced/requirements.txt @@ -3,13 +3,9 @@ astroquery >= 0.4.7 crds glob2 jdaviz -json jwst >= 1.12.5 matplotlib numpy os pandas shutil -warnings -urllib -zipfile From 48acf8717b51f44e1d21743c9c0523bdb107d408 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 11:34:01 -0400 Subject: [PATCH 47/62] Missing a couple more built-in dependencies --- notebooks/NIRISS_WFSS_advanced/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/requirements.txt b/notebooks/NIRISS_WFSS_advanced/requirements.txt index ebed5dd67..dbb55021e 100644 --- a/notebooks/NIRISS_WFSS_advanced/requirements.txt +++ b/notebooks/NIRISS_WFSS_advanced/requirements.txt @@ -6,6 +6,4 @@ jdaviz jwst >= 1.12.5 matplotlib numpy -os pandas -shutil From d1ed96c2aea7ed671089f06756e029c58ca5d100 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 11:37:08 -0400 Subject: [PATCH 48/62] More pep8 fixes --- .../00_niriss_mast_query_data_setup.ipynb | 26 +++++++++---------- .../01_niriss_wfss_image2_image3.ipynb | 10 +++---- .../02_niriss_wfss_spec2.ipynb | 4 +-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index f3bbae269..5821f3888 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -114,9 +114,9 @@ "obs_table = Observations.query_criteria(obs_collection=[\"JWST\"], \n", " instrument_name=[\"NIRISS/IMAGE\", \"NIRISS/WFSS\"],\n", " provenance_name=[\"CALJWST\"], # Executed observations\n", - " filters=['F115W','F150W','F200W'],\n", + " filters=['F115W', 'F150W', 'F200W'],\n", " proposal_id=[2079],\n", - " )\n", + " )\n", "\n", "print(len(obs_table), 'files found')\n", "# look at what was obtained in this query for a select number of column names of interest\n", @@ -146,7 +146,7 @@ "obs_id_table = Observations.query_criteria(instrument_name=[\"NIRISS*\"],\n", " provenance_name=[\"CALJWST\"], # Executed observations\n", " obs_id=['jw02079-o004*'], # Searching for PID 2079 observation 004\n", - " ) \n", + " ) \n", "\n", "print(len(obs_id_table), 'files found') # 611 files" ] @@ -229,17 +229,17 @@ "source": [ "# creating a dictionary of the above information to use for inspection of the filtering function\n", "file_dict = {'uncal': {'product_type': 'SCIENCE',\n", - " 'productSubGroupDescription': 'UNCAL',\n", - " 'calib_level': [1]},\n", + " 'productSubGroupDescription': 'UNCAL',\n", + " 'calib_level': [1]},\n", " 'rate': {'product_type': 'SCIENCE',\n", - " 'productSubGroupDescription': 'RATE',\n", - " 'calib_level': [2]},\n", + " 'productSubGroupDescription': 'RATE',\n", + " 'calib_level': [2]},\n", " 'level2_association': {'product_type': 'INFO',\n", - " 'productSubGroupDescription': 'ASN',\n", - " 'calib_level': [2]},\n", + " 'productSubGroupDescription': 'ASN',\n", + " 'calib_level': [2]},\n", " 'level3_association': {'product_type': 'INFO',\n", - " 'productSubGroupDescription': 'ASN',\n", - " 'calib_level': [3]},\n", + " 'productSubGroupDescription': 'ASN',\n", + " 'calib_level': [3]},\n", " }" ] }, @@ -271,7 +271,7 @@ " print(filtered_products['productFilename'])\n", " files_to_download.extend(filtered_products['productFilename'])\n", " print()\n", - " print('*'*50)\n" + " print('*'*50)" ] }, { @@ -352,7 +352,7 @@ " productType=[\"SCIENCE\", \"INFO\"],\n", " productSubGroupDescription=[\"RATE\", \"ASN\"], # Not using \"UNCAL\" here since we can start with the rate files\n", " calib_level=[2, 3] # not using 1 here since not getting the UNCAL files\n", - " )\n", + " )\n", " \n", " # Download products for these records.\n", " # The `flat=True` option stores all files in a single directory specified by `download_dir`.\n", diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index c013ab236..1e136f11e 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -363,7 +363,7 @@ "\n", "# the number of image3 association files should match the number of unique blocking filters used\n", "uniq_filters = np.unique(rate_df[rate_df['FILTER'] == 'CLEAR']['PUPIL'])\n", - "print(f\"{len(uniq_filters)} unique filters used: {uniq_filters}\")\n" + "print(f\"{len(uniq_filters)} unique filters used: {uniq_filters}\")" ] }, { @@ -710,14 +710,14 @@ " 'snr_threshold': 10.0,\n", " 'npixels': 50,\n", " 'deblend': True,\n", - " },\n", + " },\n", " 'tweakreg': {'snr_threshold': 20,\n", " 'abs_refcat': 'GAIADR3',\n", " 'save_catalogs': True,\n", " 'searchrad': 3.0,\n", " 'kernel_fwhm': 2.302,\n", " 'fitgeometry': 'shift',\n", - " },\n", + " },\n", " },\n", " save_results=True,\n", " output_dir=custom_run_image3)" @@ -814,14 +814,14 @@ " 'snr_threshold': 10.0,\n", " 'npixels': 50,\n", " 'deblend': True,\n", - " },\n", + " },\n", " 'tweakreg': {'snr_threshold': 20,\n", " 'abs_refcat': 'GAIADR3',\n", " 'save_catalogs': True,\n", " 'searchrad': 3.0,\n", " 'kernel_fwhm': 2.302,\n", " 'fitgeometry': 'shift',\n", - " },\n", + " },\n", " },\n", " save_results=True,\n", " output_dir=custom_run_image3)" diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index e21575552..10f180933 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -611,7 +611,7 @@ " \n", " # look for cal extension, too, but only in the SCI extension; \n", " # fill in with a source ID of -999 for all other extensions to get the right extension value\n", - " cal_source_ids = np.array([cal_hdu[ext].header['SOURCEID'] if cal_hdu[ext].header['EXTNAME'] == 'SCI' \\\n", + " cal_source_ids = np.array([cal_hdu[ext].header['SOURCEID'] if cal_hdu[ext].header['EXTNAME'] == 'SCI'\n", " else -999 for ext in range(len(cal_hdu))[1:-1]]) \n", " wh_cal = np.where(cal_source_ids == source_id)[0][0] + 1 # need to add 1 for the primary header\n", "\n", @@ -774,7 +774,7 @@ " print(x1d_file, ': x1d file already exists.')\n", "else:\n", " spec2 = Spec2Pipeline.call(spec2_asns[0],\n", - " steps={'extract_2d': {'wfss_nbright': 10,}, },\n", + " steps={'extract_2d': {'wfss_nbright': 10}, },\n", " save_results=True,\n", " output_dir=param_outdir)" ] From d8dca3e057419aeb1bf2bbe512ea4dc70468b27d Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 18:00:52 -0400 Subject: [PATCH 49/62] Adding box download to extra steps notebook --- .../extra_niriss_individual_steps.ipynb | 57 +++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb index c1974886e..219015db1 100644 --- a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb @@ -11,7 +11,7 @@ "\n", "**Use case**: When using a package outside of the standard JWST pipeline, there may be certain steps that are still helpful to utilize within the JWST pipeline.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: astropy, crds, glob, jwst, matplotlib, numpy, os, pandas
\n", + "**Tools**: astropy, crds, glob, jwst, matplotlib, numpy, os, pandas, urllib, zipfile
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -45,6 +45,8 @@ "source": [ "import os\n", "import glob\n", + "import urllib\n", + "import zipfile\n", "import numpy as np\n", "import pandas as pd\n", "\n", @@ -100,28 +102,61 @@ { "cell_type": "code", "execution_count": null, - "id": "3b9e37a5-d867-4047-9e22-ceb42a0e8a61", + "id": "48baec42-7aee-4731-8048-1054c9a127f4", "metadata": {}, "outputs": [], "source": [ - "# From the csv file we created earlier, find a list of all of the grism observations we will want to calibrate with spec2\n", - "listrate_file = 'list_ngdeep_rate.csv'\n", - "rate_df = pd.read_csv(listrate_file)" + "data_dir_in = 'data' # directory where the rate files should be\n", + "data_dir_out = 'data/calibrated_steps/' # directory where to save the calibrate files\n", + "\n", + "# if the directory does not exist that you want to save out to, make that directory first\n", + "for datadir in [data_dir_in, data_dir_out]:\n", + " if not os.path.exists(datadir):\n", + " os.makedirs(datadir)" ] }, { "cell_type": "code", "execution_count": null, - "id": "48baec42-7aee-4731-8048-1054c9a127f4", + "id": "21167b5c-3654-415d-ba9f-b5484e367c57", "metadata": {}, "outputs": [], "source": [ - "all_rate_files = rate_df['FILENAME']\n", - "data_dir_out = 'data/calibrated_steps/' # directory where to save the calibrate files\n", + "# Download uncalibrated data from Box into the data directory:\n", + "boxlink = 'https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/niriss_wfss_advanced/niriss_wfss_extra_input.zip'\n", + "boxfile = os.path.basename(boxlink)\n", + "urllib.request.urlretrieve(boxlink, boxfile)\n", "\n", - "# if the directory does not exist that you want to save out to, make that directory first\n", - "if not os.path.exists(data_dir_out):\n", - " os.makedirs(data_dir_out)" + "zf = zipfile.ZipFile(boxfile, 'r')\n", + "zf.extractall(path=data_dir_in)\n", + "\n", + "# move the files downloaded from the box file into the top level data directory\n", + "box_download_dir = os.path.join(data_dir_in, boxfile.split('.zip')[0])\n", + "for filename in glob.glob(os.path.join(box_download_dir, '*')):\n", + " if '.csv' in filename:\n", + " # move to the current directory\n", + " os.rename(filename, os.path.basename(filename))\n", + " else:\n", + " # move to the data directory \n", + " os.rename(filename, os.path.join(data_dir_in, os.path.basename(filename)))\n", + "\n", + "# remove unnecessary files now\n", + "os.remove(boxfile)\n", + "os.rmdir(box_download_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b9e37a5-d867-4047-9e22-ceb42a0e8a61", + "metadata": {}, + "outputs": [], + "source": [ + "# From the csv file we created earlier, find a list of all of the grism observations we will want to calibrate with spec2\n", + "listrate_file = 'list_ngdeep_rate.csv'\n", + "rate_df = pd.read_csv(listrate_file)\n", + "\n", + "all_rate_files = rate_df['FILENAME']" ] }, { From 6bdd5da5a3ddc207d893830bcce4ce2150c223b3 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 18:39:55 -0400 Subject: [PATCH 50/62] Removing files to try and free up space for run --- .../01_niriss_wfss_image2_image3.ipynb | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 1e136f11e..eb7e64186 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -439,6 +439,18 @@ " img3 = Image3Pipeline.call(img3_asn, save_results=True, output_dir=default_run_image3)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb5b3614-cfc3-42b7-b5ce-e243eb680ae7", + "metadata": {}, + "outputs": [], + "source": [ + "# remove unnecessary files to save disk space\n", + "for crf in glob.glob(os.path.join(default_run_image3, '*crf.fits')):\n", + " os.remove(crf)" + ] + }, { "cell_type": "markdown", "id": "33f0dbeb-4caf-4d97-8b2c-3255f906e76b", @@ -723,6 +735,18 @@ " output_dir=custom_run_image3)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "403345ea-1f26-4ead-b680-ab6a54f73421", + "metadata": {}, + "outputs": [], + "source": [ + "# remove unnecessary files to save disk space\n", + "for crf in glob.glob(os.path.join(custom_run_image3, '*crf.fits')):\n", + " os.remove(crf)" + ] + }, { "cell_type": "markdown", "id": "17d4cdae-0cad-4385-b93b-6976ecf90898", @@ -827,6 +851,18 @@ " output_dir=custom_run_image3)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "828566bf-fb5f-4ba5-8ef7-0de09a73f0fb", + "metadata": {}, + "outputs": [], + "source": [ + "# remove unnecessary files to save disk space\n", + "for crf in glob.glob(os.path.join(custom_run_image3, '*crf.fits')):\n", + " os.remove(crf)" + ] + }, { "cell_type": "code", "execution_count": null, From 4210642a409abca134953a48e755d87fb46b1270 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 18:59:45 -0400 Subject: [PATCH 51/62] defining CRDS --- .../extra_niriss_individual_steps.ipynb | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb index 219015db1..46b500809 100644 --- a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb @@ -36,6 +36,18 @@ "## Imports & Data Setup" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "54802ba1-873a-4a62-9db1-84f60d1677e0", + "metadata": {}, + "outputs": [], + "source": [ + "# Update the CRDS path to your local directory\n", + "%env CRDS_PATH=crds_cache\n", + "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" + ] + }, { "cell_type": "code", "execution_count": null, @@ -59,16 +71,6 @@ "from jwst.flatfield import FlatFieldStep" ] }, - { - "cell_type": "raw", - "id": "f0e5031b-f273-47d6-97b6-41e0fd2d3a47", - "metadata": {}, - "source": [ - "# Update the CRDS path to your local directory\n", - "%env CRDS_PATH= (/grp/crds/jwst/)\n", - "%env CRDS_SERVER_URL=https://jwst-crds.stsci.edu" - ] - }, { "cell_type": "markdown", "id": "3c208323-56f4-4fb5-b8c0-750b65b525aa", From 9970612a0e394ff6e36f2f5bd1ae89a9e334edb5 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 22 Mar 2024 19:53:22 -0400 Subject: [PATCH 52/62] Only calibrating 1 dataset for the extra notebook b/c is all that is really needed for the point --- .../extra_niriss_individual_steps.ipynb | 31 +++++-------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb index 46b500809..1bdf6f30f 100644 --- a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb @@ -11,7 +11,7 @@ "\n", "**Use case**: When using a package outside of the standard JWST pipeline, there may be certain steps that are still helpful to utilize within the JWST pipeline.
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", - "**Tools**: astropy, crds, glob, jwst, matplotlib, numpy, os, pandas, urllib, zipfile
\n", + "**Tools**: astropy, crds, glob, jwst, matplotlib, numpy, os, urllib, zipfile
\n", "**Cross-instrument**: NIRISS
\n", "\n", "**Content**\n", @@ -60,7 +60,6 @@ "import urllib\n", "import zipfile\n", "import numpy as np\n", - "import pandas as pd\n", "\n", "from astropy.io import fits\n", "\n", @@ -147,20 +146,6 @@ "os.rmdir(box_download_dir)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "3b9e37a5-d867-4047-9e22-ceb42a0e8a61", - "metadata": {}, - "outputs": [], - "source": [ - "# From the csv file we created earlier, find a list of all of the grism observations we will want to calibrate with spec2\n", - "listrate_file = 'list_ngdeep_rate.csv'\n", - "rate_df = pd.read_csv(listrate_file)\n", - "\n", - "all_rate_files = rate_df['FILENAME']" - ] - }, { "cell_type": "markdown", "id": "d9bfb726-26b5-46c1-99bf-3c86e0c9e3bb", @@ -198,9 +183,9 @@ }, "outputs": [], "source": [ - "# Run assign_wcs\n", - "for ratefile in all_rate_files:\n", - " result = AssignWcsStep.call(ratefile, output_dir=data_dir_out, save_results=True)" + "# Run assign_wcs; we are only running on one file for demonstration here\n", + "ratefile = os.path.join(data_dir_in, 'jw02079004002_02101_00001_nis_rate.fits')\n", + "result = AssignWcsStep.call(ratefile, output_dir=data_dir_out, save_results=True)" ] }, { @@ -213,7 +198,7 @@ "# A quick sanity check to ensure that the files were calibrated.\n", "# if this is zero, check the log message above for any errors that may have occurred during the calibration\n", "wcsstep_files = glob.glob(os.path.join(data_dir_out, '*assignwcsstep*'))\n", - "print(len(wcsstep_files), 'assignwcsstep files written') # 29 files should have been written" + "print(len(wcsstep_files), 'assignwcsstep files written') # 1 file should have been written" ] }, { @@ -251,7 +236,7 @@ "# A quick sanity check to ensure that the files were calibrated.\n", "# if this is zero, check the log message above for any errors that may have occurred during the calibration\n", "flatfield_files = glob.glob(os.path.join(data_dir_out, '*flatfieldstep*'))\n", - "print(len(flatfield_files), 'flatfieldstep files written') # 29 files should have been written (matching the wcs step)" + "print(len(flatfield_files), 'flatfieldstep files written') # 1 file should have been written (matching the wcs step)" ] }, { @@ -263,7 +248,7 @@ "#### Compare Rate vs. Flat fielded Data\n", "Running the cell below shows the same direct image from just the rate file versus when the `flat_field` step of the pipeline is run. Some detector artifacts are noticably gone.\n", "\n", - "There are remaining optical artifacts to be aware of: https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph/niriss-instrument-features-and-caveats" + "There are remaining optical artifacts to be aware of: https://jwst-docs.stsci.edu/known-issues-with-jwst-data/niriss-known-issues" ] }, { @@ -273,7 +258,7 @@ "metadata": {}, "outputs": [], "source": [ - "test_rate_file = np.sort(list(rate_df[rate_df['FILTER'] == 'CLEAR']['FILENAME']))[0]\n", + "test_rate_file = ratefile # look at a direct image for this comparision\n", "test_flat_file = os.path.join(data_dir_out, os.path.basename(test_rate_file).replace('rate.fits', 'flatfieldstep.fits'))\n", "\n", "plot_files = [test_rate_file, test_flat_file]\n", From 2fd5732292b61630db028c61a97e4454978c6486 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 25 Apr 2024 12:45:02 -0400 Subject: [PATCH 53/62] Make the version 0.4.7 requirement a bit more obvious to try and help with crashes --- .../00_niriss_mast_query_data_setup.ipynb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb index 5821f3888..01f5e6afa 100644 --- a/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/00_niriss_mast_query_data_setup.ipynb @@ -25,7 +25,7 @@ "\n", "\n", "**Author**: Camilla Pacifici (cpacifici@stsci.edu) & Rachel Plesha (rplesha@stsci.edu) & Jo Taylor (jotaylor@stsci.edu)
\n", - "**First Published**: February 2024\n", + "**First Published**: May 2024\n", "\n", "This notebook was inspired by the [JWebbinar session about MAST](https://github.com/spacetelescope/jwebbinar_prep/blob/main/mast_session/Crowded_Field/Crowded_Field.ipynb)." ] @@ -148,7 +148,8 @@ " obs_id=['jw02079-o004*'], # Searching for PID 2079 observation 004\n", " ) \n", "\n", - "print(len(obs_id_table), 'files found') # 611 files" + "# this number will change with JWST pipeline and reference file updates\n", + "print(len(obs_id_table), 'files found') # ~613 files" ] }, { @@ -292,7 +293,19 @@ "To make create a new API token visit to following link: \n", " https://auth.mast.stsci.edu/token?suggested_name=Astroquery&suggested_scope=mast:exclusive_access\n", "\n", - "*Note that a developmental version of astroquery is currently required to have the call `flat=True` when downloading the data. If you prefer to not use a developmental version, remove that line from the call, download the data, and move all of the files in all downloaded subfolders into the single directory as defined by the `download_dir` variable.*" + "*Note that a version of astroquery >= 0.4.7 is required to have the call `flat=True` when downloading the data. If you prefer to use an earlier version, remove that line from the call, download the data, and move all of the files in all downloaded subfolders into the single directory as defined by the `download_dir` variable.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9ca7055-81bd-4a94-a766-45bb463bc0fa", + "metadata": {}, + "outputs": [], + "source": [ + "# check that the version is above 0.4.7. See above note for more information\n", + "import astroquery\n", + "astroquery.__version__" ] }, { From 8d7bf7f00518e11c76538869fe462db76926042a Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 25 Apr 2024 13:42:30 -0400 Subject: [PATCH 54/62] Fixing imviz bug with linking with the latest version --- .../01_niriss_wfss_image2_image3.ipynb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index eb7e64186..9dd795b3c 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -627,10 +627,6 @@ "imviz = Imviz()\n", "viewer = imviz.default_viewer\n", "\n", - "# this aligns the image to use the WCS coordinates\n", - "linking = imviz.plugins['Links Control']\n", - "linking.link_type = 'WCS'\n", - "\n", "# plot each i2d image\n", "catalogs = [] # for plotting the catalogs\n", "labels = [] # for plotting the catalogs\n", @@ -645,6 +641,10 @@ " catalogs.append(img.replace('i2d.fits', 'cat.ecsv'))\n", " labels.append(label)\n", "\n", + "# this aligns the image to use the WCS coordinates; \n", + "# the images need to be loaded first, but before adding markers\n", + "imviz.link_data(link_type='wcs')\n", + "\n", "# also plot the associated catalog\n", "# this needs to be a separate loop due to linking in imviz when using sky coordinates\n", "for catname, label in zip(catalogs, labels):\n", @@ -770,10 +770,18 @@ "compare_fig = plot_image_and_segmentation_map(compare_i2ds, compare_segm)" ] }, + { + "cell_type": "markdown", + "id": "8fd7891a-22ec-4913-a9fd-b153b66a932e", + "metadata": {}, + "source": [ + "The cell below shows similar information using Imviz instead to visualize this." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "1ac548eb-ae6a-4c08-ba87-8d5714a9880b", + "id": "724d7a9e-f923-4cc0-8e99-6d21008db42a", "metadata": {}, "outputs": [], "source": [ From b5c6e0b662e0ec7d34772e80353bdfd0e180b376 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 25 Apr 2024 17:09:58 -0400 Subject: [PATCH 55/62] Created a new figure overplotting the different grisms --- .../01_niriss_wfss_image2_image3.ipynb | 4 +- .../02_niriss_wfss_spec2.ipynb | 118 ++++++++++++++++-- 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 9dd795b3c..76019126a 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -28,9 +28,9 @@ " - [Manually Editing the Source Catalog](#manual_cat)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", - "**First Published**: March 2024\n", + "**First Published**: May 2024\n", "\n", - "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1215.pmap." + "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1225.pmap." ] }, { diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index 10f180933..99d259207 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -26,9 +26,9 @@ " - [Final Visualization](#final_visualize)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", - "**First Published**: March 2024\n", + "**First Published**: May 2024\n", "\n", - "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1215.pmap." + "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1225.pmap." ] }, { @@ -81,7 +81,8 @@ "\n", "from matplotlib import pyplot as plt\n", "from matplotlib import patches\n", - "%matplotlib inline\n", + "%matplotlib widget\n", + "#%matplotlib inline\n", "\n", "from jwst.pipeline import Spec2Pipeline" ] @@ -704,7 +705,7 @@ "outputs": [], "source": [ "def plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file, source_id):\n", - " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 5))\n", + " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 5))\n", " \n", " # plot the cal image\n", " ax1.imshow(cal_data, origin='lower', vmin=0, vmax=np.nanmax(cal_data)*.01, aspect='auto')\n", @@ -721,9 +722,16 @@ " max_flux = np.nanmax(flux[edge_buffer:edge_buffer*-1])\n", " ax2.set_ylim(0, max_flux+(max_flux*0.1)) # cutting the flux of the edges & adding 10% buffer to the limits\n", " ax2.set_xlabel('Wavelength (Microns)')\n", - " ax2.set_ylabel('Flux')\n", + " ax2.set_ylabel('Flux (Jy)')\n", + "\n", + " if fits.getval(cal_file, 'FILTER') == 'GR150C':\n", + " ax1.set_xlabel('X Pixels (<--- dispersion direction)')\n", + " ax1.set_ylabel('Y Pixels')\n", + " else:\n", + " ax1.set_xlabel('X Pixels')\n", + " ax1.set_ylabel('Y Pixels (<--- dispersion direction)') \n", " \n", - " plt.suptitle(f\"{fits.getval(cal_file, 'FILTER')} {fits.getval(cal_file, 'PUPIL')} Source {source_id}\")" + " plt.suptitle(f\"{fits.getval(cal_file, 'FILTER')} {fits.getval(cal_file, 'PUPIL')} Source {source_id}\", x=0.5, y=1)" ] }, { @@ -986,7 +994,9 @@ "\n", "### Final Visualization\n", "\n", - "Now that everything has been calibrated, it's useful to look at all of the extracted files. The cal and x1d files from spec2 are extracted at each dither step, which is shown below. Spec3 then turns the individual dither x1d files into a single combined spectrum per grism and filter for each source." + "Now that everything has been calibrated, it's useful to look at all of the extracted files. The cal and x1d files from spec2 are extracted at each dither step, which is shown below. Spec3 then turns the individual dither x1d files into a single combined spectrum per grism and filter for each source.\n", + "\n", + "Note that for GR150R data, the dispersion direction is in the -Y direction, and for GR150C data, the dispersion direction is in the -X direction." ] }, { @@ -1028,12 +1038,98 @@ " plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file, source_id)" ] }, + { + "cell_type": "markdown", + "id": "08135225-8662-4b89-8b2d-e422ae087e73", + "metadata": {}, + "source": [ + "Overplot these files on top of each other to compare. The two grisms will be different line styles to draw attention to any differences that could be due to the calibration, including contamination, and each blocking filter will be a different color." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0fc37eb-a7c8-4083-82a6-80e329f1fa3c", + "metadata": {}, + "outputs": [], + "source": [ + "# Overplot the different grisms on top of each other for the same source\n", + "x1ds = np.sort(glob.glob(os.path.join(source_outdir, \"*x1d.fits\")))\n", + "\n", + "# get a list of all of the source IDs from the first file to look at for this example\n", + "sources = [fits.getval(x1ds[0], 'SOURCEID', ext=ext) for ext in range(len(fits.open(x1ds[0])))[1:-1]]\n", + "source_id = 118\n", + "\n", + "# create a figure with three panels\n", + "src118_fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(11, 9), sharex=True, sharey=True)\n", + "\n", + "ls_dict = {'GR150R': '--',\n", + " 'GR150C': '-',\n", + " }\n", + "\n", + "color_dict = {'F115W': '#e1cb00',\n", + " 'F150W': '#32b45c',\n", + " 'F200W': '#099ab4',\n", + " }\n", + "\n", + "max_fluxes = []\n", + "all_waves = []\n", + "# plot each x1d file\n", + "for i, x1d_file in enumerate(x1ds):\n", + " with fits.open(x1d_file) as x1d_hdu:\n", + " try:\n", + " wh_x1d, wh_cal = find_source_ext(x1d_hdu, cal_hdu, source_id, info=False)\n", + " except IndexError:\n", + " # this means the source isn't in this observation\n", + " continue\n", + "\n", + " x1d_data = x1d_hdu[wh_x1d].data \n", + " grism = x1d_hdu[0].header['FILTER']\n", + " filter = x1d_hdu[0].header['PUPIL']\n", + "\n", + " wave = x1d_data['WAVELENGTH'].ravel()\n", + " flux = x1d_data['FLUX'].ravel()\n", + "\n", + " ax1.plot(wave, flux, color=color_dict[filter], ls=ls_dict[grism], alpha=0.7)\n", + " if grism == 'GR150C':\n", + " ax2.plot(wave, flux, color=color_dict[filter], ls=ls_dict[grism], alpha=0.7)\n", + " else:\n", + " ax3.plot(wave, flux, color=color_dict[filter], ls=ls_dict[grism], alpha=0.7)\n", + "\n", + " # save the maximum fluxes for plotting, removing any edge effects\n", + " edge_buffer = int(len(flux) * .25)\n", + " max_fluxes.append(np.nanmax(flux[edge_buffer:edge_buffer*-1]))\n", + " all_waves.extend(wave)\n", + "\n", + "# plot limits & labels\n", + "ax1.set_ylim(0, np.max(max_fluxes))\n", + "ax1.set_xlim(np.min(all_waves), np.max(all_waves))\n", + "\n", + "src118_fig.suptitle(f'Source {source_id}')\n", + "ax1.set_title('GR150R & GR150C')\n", + "ax2.set_title('GR150C')\n", + "ax3.set_title('GR150R')\n", + "\n", + "for ax in [ax1, ax2, ax3]:\n", + " ax.set_xlabel('Wavelength (Microns)')\n", + " ax.set_ylabel('Flux (Jy)')\n", + "\n", + "# label for each of the filters\n", + "for filt, color in color_dict.items():\n", + " ax1.plot(0, 0, color=color, label=filt)\n", + "\n", + "ax1.legend(bbox_to_anchor=(1, 1.05))\n", + "src118_fig.tight_layout()" + ] + }, { "cell_type": "markdown", "id": "123375a9-4660-4205-9128-4b077bfc06dc", "metadata": {}, "source": [ - "##### Look at all of the sources for a single file" + "##### Look at all of the sources for a single file\n", + "\n", + "Note that some sources might not actually be extracting anything interesting. If this is the case, go back to your source catalog and images to ensure that you have the correct source identified and the target is centered in the cutout. This notebook uses simple examples to create and limit the source catalog, so it might not always show the most scientifically interesting sources." ] }, { @@ -1079,7 +1175,7 @@ "outputs": [], "source": [ "# setting up the figure\n", - "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 7))\n", + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 5), sharex=True, sharey=True)\n", "\n", "# **grism data\n", "\n", @@ -1121,7 +1217,7 @@ " \n", " rectangle = patches.Rectangle((sx_left, sy_bottom), swidth, sheight, edgecolor='darkred', facecolor=\"None\", linewidth=1)\n", " ax1.add_patch(rectangle)\n", - " ax1.text(sx_left, sy_top+10, source_id, fontsize=14, color='darkred')\n", + " ax1.text(sx_left, sy_top+10, source_id, fontsize=12, color='darkred')\n", " \n", " ax1.set_title(f\"{os.path.basename(x1d_file).split('_nis')[0]}: {cal_hdu[0].header['FILTER']} {cal_hdu[0].header['PUPIL']}\")\n", "\n", @@ -1146,7 +1242,7 @@ " for i, source_num in enumerate(sources['label']):\n", " ax2.annotate(source_num, \n", " (sources['xcentroid'][i]+0.5, sources['ycentroid'][i]+0.5), \n", - " fontsize=14,\n", + " fontsize=12,\n", " color=color)\n", "\n", "fig.tight_layout()" From dde80cdd0001cd8b83700f17bddbaf6c7b8ebc5a Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Thu, 25 Apr 2024 17:15:31 -0400 Subject: [PATCH 56/62] pep8 failure fix --- notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index 99d259207..d12e313e7 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -82,7 +82,7 @@ "from matplotlib import pyplot as plt\n", "from matplotlib import patches\n", "%matplotlib widget\n", - "#%matplotlib inline\n", + "# %matplotlib inline\n", "\n", "from jwst.pipeline import Spec2Pipeline" ] From 99dd82bf40c99393c1e4787bc9556130c8633467 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 6 May 2024 11:54:12 -0400 Subject: [PATCH 57/62] Updating documentation in extra notebook based on feedback --- .../extra_niriss_individual_steps.ipynb | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb index 1bdf6f30f..e8d5d0b32 100644 --- a/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/extra_niriss_individual_steps.ipynb @@ -7,9 +7,9 @@ "source": [ "# Running Individual Pipeline Steps\n", "\n", - "This notebook walks through calibrating the data with individual pipeline steps rather than running the entire pipeline stage.\n", + "This notebook walks through calibrating the data with individual pipeline steps (AssignWCS and FlatField) rather than running the entire pipeline stage.\n", "\n", - "**Use case**: When using a package outside of the standard JWST pipeline, there may be certain steps that are still helpful to utilize within the JWST pipeline.
\n", + "**Use case**: When using a package outside of the standard JWST pipeline, there may be certain steps that are still helpful to utilize within the JWST pipeline. Here we show the most commonly run individual steps, AssignWCS and FlatField. AssignWCS determines and stores the WCS (World Coordinate System) information, and FlatField removes [detector features](https://jwst-docs.stsci.edu/known-issues-with-jwst-data/niriss-known-issues#NIRISSKnownIssues-artifactsArtifacts).
\n", "**Data**: JWST/NIRISS images and spectra from program 2079.
\n", "**Tools**: astropy, crds, glob, jwst, matplotlib, numpy, os, urllib, zipfile
\n", "**Cross-instrument**: NIRISS
\n", @@ -22,9 +22,8 @@ " - [Compare Rate vs. Flat Fielded Data](#compare)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu)
\n", - "**Last modified**: March 2024\n", - "\n", - "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1215.pmap." + "**Last modified**: May 2024
\n", + "**Last tested**: This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1229.pmap." ] }, { @@ -75,7 +74,7 @@ "id": "3c208323-56f4-4fb5-b8c0-750b65b525aa", "metadata": {}, "source": [ - "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available or install a previous version, check [GitHub](https://github.com/spacetelescope/jwst#software-vs-dms-build-version-map\"). Also verify what [CRDS version](https://jwst-crds.stsci.edu/) you are using. [CRDS documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/user_documentation/reference_files_crds.html) explains how to set a specific context to use in the JWST pipeline. If either of these values are different from the last tested note above there may be differences in this notebook." + "Check what version of the JWST pipeline you are using. To see what the latest version of the pipeline is available or install a previous version, check [GitHub](https://github.com/spacetelescope/jwst#software-vs-dms-build-version-map). Also verify what [CRDS context](https://jwst-crds.stsci.edu/) you are using. [CRDS documentation](https://jwst-pipeline.readthedocs.io/en/latest/jwst/user_documentation/reference_files_crds.html) explains how to set a specific context to use in the JWST pipeline. If either of these values are different from the last tested note above there may be differences in this notebook." ] }, { @@ -97,7 +96,7 @@ "metadata": {}, "source": [ "#### Data setup\n", - "Open the file that contains all of the rate files, and create an output directory for the calibrated files if it does not already exist" + "Here we download and open the zip file that contains all of the rate files, and we also create an output directory for the calibrated files if it does not already exist" ] }, { @@ -171,7 +170,7 @@ "\n", "#### Assign WCS Step\n", "\n", - "The `assign_wcs` step of the pipeline is a critical part to obtaining the correct spectral trace cutouts for WFSS images. To read more about the step, visit the [AssignWCS jdox page](https://jwst-pipeline.readthedocs.io/en/latest/jwst/assign_wcs/main.html)." + "The `assign_wcs` step of the pipeline is a critical part to obtaining the correct spectral trace cutouts for WFSS images. To read more about the step, visit the [AssignWCS description page](https://jwst-pipeline.readthedocs.io/en/latest/jwst/assign_wcs/main.html)." ] }, { @@ -209,7 +208,7 @@ "\n", "#### Flat Field Step\n", "\n", - "After the assignwcs file is run, we then want to run the `flat_field` step of the pipeline which removes detector artifacts using the flat field reference files. Read more about the step on [jdox](https://jwst-pipeline.readthedocs.io/en/latest/jwst/flatfield/main.html)" + "After the assignwcs file is run, we then want to run the `flat_field` step of the pipeline which removes detector artifacts using the flat field reference files. To read more about the step, visit the [FlatField description page](https://jwst-pipeline.readthedocs.io/en/latest/jwst/flatfield/main.html)." ] }, { @@ -246,9 +245,9 @@ "source": [ "\n", "#### Compare Rate vs. Flat fielded Data\n", - "Running the cell below shows the same direct image from just the rate file versus when the `flat_field` step of the pipeline is run. Some detector artifacts are noticably gone.\n", + "Running the cell below shows the same direct image from just the rate file versus when the `flat_field` step of the pipeline is run. Some detector artifacts are noticably gone such as the [cross-hatching](https://jwst-docs.stsci.edu/known-issues-with-jwst-data/niriss-known-issues#NIRISSKnownIssues-Cross-hatching) in the bottom right and middle of the detector.\n", "\n", - "There are remaining optical artifacts to be aware of: https://jwst-docs.stsci.edu/known-issues-with-jwst-data/niriss-known-issues" + "There are remaining optical artifacts to be aware of on the [NIRISS known issues page](https://jwst-docs.stsci.edu/known-issues-with-jwst-data/niriss-known-issues) such as the 1/f noise." ] }, { @@ -273,7 +272,7 @@ " data = hdu[1].data\n", " data[np.isnan(data)] = 0\n", " \n", - " ax.imshow(data, vmin=0, vmax=1.7, origin='lower')\n", + " ax.imshow(data, vmin=0.2, vmax=1.2, origin='lower')\n", " ax.set_title(title)" ] }, From a241442cbcace6a6d5ae9df7b0e5242055ac6c02 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 6 May 2024 11:55:08 -0400 Subject: [PATCH 58/62] Updating notebooks to match the 'last tested' addition to the extra notebook --- .../NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb | 5 ++--- notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 76019126a..48382e2fd 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -28,9 +28,8 @@ " - [Manually Editing the Source Catalog](#manual_cat)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", - "**First Published**: May 2024\n", - "\n", - "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1225.pmap." + "**First Published**: May 2024
\n", + "**Last tested**: This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1225.pmap." ] }, { diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index d12e313e7..0d8d75031 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -26,9 +26,8 @@ " - [Final Visualization](#final_visualize)\n", "\n", "**Author**: Rachel Plesha (rplesha@stsci.edu), Camilla Pacifici (cpacifici@stsci.edu), JWebbinar notebooks.
\n", - "**First Published**: May 2024\n", - "\n", - "This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1225.pmap." + "**First Published**: May 2024
\n", + "**Last tested**: This notebook was last tested with JWST pipeline version 1.12.5 and the CRDS context jwst_1225.pmap." ] }, { From 8098a63ed4834ca6959dbc3e78bcaa4a19f8f7b2 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 6 May 2024 16:33:04 -0400 Subject: [PATCH 59/62] Adding a quick look at the galaxy we're planning to extract later --- .../01_niriss_wfss_image2_image3.ipynb | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 48382e2fd..40fe4865c 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -84,7 +84,8 @@ "\n", "from jdaviz import Imviz\n", "from matplotlib import pyplot as plt\n", - "%matplotlib inline\n", + "%matplotlib widget\n", + "#%matplotlib inline\n", "\n", "from jwst.pipeline import Image2Pipeline\n", "from jwst.pipeline import Image3Pipeline" @@ -1127,7 +1128,50 @@ "id": "9d2a8566-eb1c-4df0-b4eb-f24c6edc8e56", "metadata": {}, "source": [ - "Once we have an updated source catalog that we are content with, we can move on to the spec2 step of the pipeline. It likely will be necessary to come back to this step after running spec2." + "Once we have an updated source catalog that we are content with, we can move on to the spec2 step of the pipeline. It likely will be necessary to come back to this step after running spec2. Let's take a quick look at the source that we will be extracting in the following notebook with spec2." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "443842e0-b30a-4a31-937c-1880eb7f697a", + "metadata": {}, + "outputs": [], + "source": [ + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(9, 6))\n", + "\n", + "img = cust_image3_i2d[-1]\n", + " \n", + "cat = Table.read(new_cat_name)\n", + "\n", + "for ax in [ax1, ax2]:\n", + " # plot the image\n", + " with fits.open(img) as hdu:\n", + " ax.imshow(hdu[1].data, vmin=0, vmax=0.3, origin='lower')\n", + " title = f\"obs{hdu[0].header['OBSERVTN']} {hdu[0].header['PUPIL']}\"\n", + " \n", + " # also plot the associated catalog\n", + " extended_sources = cat[cat['is_extended'] == 1] # 1 is True; i.e. is extended\n", + " point_sources = cat[cat['is_extended'] == 0] # 0 is False; i.e. is a point source\n", + " \n", + " for color, sources in zip(['darkred', 'black'], [extended_sources, point_sources]):\n", + " # plotting the sources\n", + " ax.scatter(sources['xcentroid'], sources['ycentroid'], s=20, facecolors='None', edgecolors=color, alpha=0.9)\n", + " \n", + " # adding source labels \n", + " for i, source_num in enumerate(sources['label']):\n", + " ax.annotate(source_num, \n", + " (sources['xcentroid'][i]+0.5, sources['ycentroid'][i]+0.5), \n", + " fontsize=8,\n", + " color=color)\n", + "\n", + "fig.suptitle(f\"Speicifc Source to Extract with Spec2\")\n", + "\n", + "# zooming in on a smaller region\n", + "ax2.set_xlim(known_x-50, known_x+50)\n", + "ax2.set_ylim(known_y-50, known_y+50)\n", + "\n", + "fig.tight_layout()" ] }, { From b369d709ed805ce5fa0748659de3ec93525e7356 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 6 May 2024 16:33:39 -0400 Subject: [PATCH 60/62] Flux unit conversion --- .../02_niriss_wfss_spec2.ipynb | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb index 0d8d75031..38ec1db11 100644 --- a/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/02_niriss_wfss_spec2.ipynb @@ -77,6 +77,7 @@ "\n", "from astropy.table import Table\n", "from astropy.io import fits\n", + "from astropy import constants as const\n", "\n", "from matplotlib import pyplot as plt\n", "from matplotlib import patches\n", @@ -669,9 +670,9 @@ "sy_top = cal_header['SLTSTRT2'] + sheight\n", "\n", "# plot the rate file and the extraction box\n", - "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 5))\n", - "ax1.imshow(rate_data, origin='lower', vmin=0, vmax=2, aspect='auto')\n", - "ax2.imshow(rate_data, origin='lower', vmin=0, vmax=2, aspect='auto')\n", + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))\n", + "ax1.imshow(rate_data, origin='lower', vmin=0.2, vmax=1, aspect='auto') # the scaling may need some adjustment\n", + "ax2.imshow(rate_data, origin='lower', vmin=0.2, vmax=1, aspect='auto') # the scaling may need some adjustment\n", "\n", "rectangle = patches.Rectangle((sx_left, sy_bottom), swidth, sheight, edgecolor='darkorange', facecolor=\"None\", linewidth=1)\n", "ax1.add_patch(rectangle)\n", @@ -691,9 +692,24 @@ "id": "22a7c98b-0a1b-4d1f-9323-25eb08696e4d", "metadata": {}, "source": [ - "We can then take a look at the extracted spectrum in this box both in the cal file and the x1d file. **Information about this source!** \n", + "We can then take a look at the extracted spectrum in this box both in the cal file and the x1d file. In the extracted spectrum below you can see the [OII] and H$\\beta$ emission lines from the galaxy.\n", "\n", - "*Note: The upturned edge effects seen in the 1-D spectrum are due to interpolation at the edges of the extraction box for the current flux calibration. This is also part of an ongoing calibration effort.*" + "*Note: The upturned edge effects seen in the 1-D spectrum are due to interpolation at the edges of the extraction box for the current flux calibration. This is also part of an ongoing calibration effort.*\n", + "\n", + "*Additional note: The default units of flux from the pipeline are in Jansky. However, in these extracted spectra we show units of erg/s/cm^2/Angstrom. To turn this off, set `convert=False` in `plot_cutout_and_spectrum`*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce6bfa92-3ba5-4863-b36f-8cd3dbb4d704", + "metadata": {}, + "outputs": [], + "source": [ + "def Fnu_to_Flam(wave_micron, flux_jansky):\n", + " # convert Jansky flux units to erg/s/cm^2/Angstrom with an input wavelength in microns\n", + " f_lambda = 1E-19 * flux_jansky * (const.c.value) / (wave_micron**2) # erg/s/cm^2/Angstom\n", + " return f_lambda" ] }, { @@ -703,7 +719,7 @@ "metadata": {}, "outputs": [], "source": [ - "def plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file, source_id):\n", + "def plot_cutout_and_spectrum(cal_data, x1d_data, cal_file, x1d_file, source_id, convert=True):\n", " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 5))\n", " \n", " # plot the cal image\n", @@ -714,6 +730,12 @@ " wave = x1d_data['WAVELENGTH'].ravel()\n", " flux = x1d_data['FLUX'].ravel()\n", "\n", + " if convert:\n", + " flux = Fnu_to_Flam(wave, flux)\n", + " fluxunit = 'egs/s/cm^2/Angstrom'\n", + " else:\n", + " fluxunit = 'Jy'\n", + "\n", " ax2.plot(wave, flux)\n", " ax2.set_title(os.path.basename(x1d_file))\n", "\n", @@ -721,7 +743,7 @@ " max_flux = np.nanmax(flux[edge_buffer:edge_buffer*-1])\n", " ax2.set_ylim(0, max_flux+(max_flux*0.1)) # cutting the flux of the edges & adding 10% buffer to the limits\n", " ax2.set_xlabel('Wavelength (Microns)')\n", - " ax2.set_ylabel('Flux (Jy)')\n", + " ax2.set_ylabel(f'Flux ({fluxunit})')\n", "\n", " if fits.getval(cal_file, 'FILTER') == 'GR150C':\n", " ax1.set_xlabel('X Pixels (<--- dispersion direction)')\n", @@ -1089,6 +1111,9 @@ " wave = x1d_data['WAVELENGTH'].ravel()\n", " flux = x1d_data['FLUX'].ravel()\n", "\n", + " flux = Fnu_to_Flam(wave, flux)\n", + " fluxunits = 'erg/s/cm^2/Angstrom'\n", + "\n", " ax1.plot(wave, flux, color=color_dict[filter], ls=ls_dict[grism], alpha=0.7)\n", " if grism == 'GR150C':\n", " ax2.plot(wave, flux, color=color_dict[filter], ls=ls_dict[grism], alpha=0.7)\n", @@ -1111,7 +1136,7 @@ "\n", "for ax in [ax1, ax2, ax3]:\n", " ax.set_xlabel('Wavelength (Microns)')\n", - " ax.set_ylabel('Flux (Jy)')\n", + " ax.set_ylabel(f'Flux ({fluxunits})')\n", "\n", "# label for each of the filters\n", "for filt, color in color_dict.items():\n", From 451fd0cd94a924a9cde9a28e83fe272a01d549e7 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Mon, 6 May 2024 16:36:46 -0400 Subject: [PATCH 61/62] pep8 style fixes --- .../NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index 40fe4865c..c90384dbe 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -85,7 +85,7 @@ "from jdaviz import Imviz\n", "from matplotlib import pyplot as plt\n", "%matplotlib widget\n", - "#%matplotlib inline\n", + "# %matplotlib inline\n", "\n", "from jwst.pipeline import Image2Pipeline\n", "from jwst.pipeline import Image3Pipeline" @@ -1165,7 +1165,7 @@ " fontsize=8,\n", " color=color)\n", "\n", - "fig.suptitle(f\"Speicifc Source to Extract with Spec2\")\n", + "fig.suptitle(\"Speicifc Source to Extract with Spec2\")\n", "\n", "# zooming in on a smaller region\n", "ax2.set_xlim(known_x-50, known_x+50)\n", From 6e740b9a156d87d289bc7e9bcb1d117acd2f23a9 Mon Sep 17 00:00:00 2001 From: Rachel Plesha Date: Fri, 24 May 2024 11:36:26 -0400 Subject: [PATCH 62/62] Minor typo fix in an error print statement --- .../NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb index c90384dbe..e49adbe82 100644 --- a/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb +++ b/notebooks/NIRISS_WFSS_advanced/01_niriss_wfss_image2_image3.ipynb @@ -700,9 +700,7 @@ "cell_type": "code", "execution_count": null, "id": "3f2f7f0f", - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "image3_asns = np.sort(glob.glob('*image3*asn*.json'))\n", @@ -713,7 +711,7 @@ "i2d_file = os.path.join(custom_run_image3, f\"{asn_data['products'][0]['name']}_i2d.fits\")\n", "\n", "if os.path.exists(i2d_file):\n", - " print(cal_file, 'cal file already exists.')\n", + " print(i2d_file, 'i2d file already exists.')\n", "else:\n", " # call the image3 pipeline in the same way as before, but add a few new modifications\n", " cust_img3 = Image3Pipeline.call(test_asn,\n",