-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* WL: average ellipticity baseline estimator, lensing catalog generation script (#1058) * Batched to_tile_catalog (#1057) batched to_tile_catalog Co-authored-by: Yicun Duan <[email protected]> * Add lensed ellipticities to tile catalog; ellipticity vs shear notebook * Organize notebooks * Script to generate DC2 pkl file with shear, conv, ellipticity, redshift * Use avg ellipticity as baseline shear estimator; fix MSE bugs; remove baseline convergence estimator * Try-except block in cached_dataset * Update catalog and splits paths in config, update plots dir name --------- Co-authored-by: Yicun Duan <[email protected]> Co-authored-by: Yicun Duan <[email protected]> * merged new workflow changes * Make lensed ellipticity compatible with new to_tile_catalog * Remove unnecessary subclass of SimulatedDataset * Add SDSS-like simulator capabilities to lensing_config * Update image generation notebook under SDSS-like simulator * updated vardist and lensing to filter ellip * Preliminary version of redshift notebook * fixed mag mask error in lensingdc2 * fixed ellip key set * Small tweaks to lensing config and ellip/redshift notebooks * New catalog merge strategy: notebook * Update variable name for r-band magnitude after new merge strategy * New catalog merge strategy: script * Run dc2_generate_lensing_catalog in order * Update splits path in config * Squeeze tile dict in lensing_dc2 (like we did before) * Don't use tract_filter when loading in object_with_truth_match * Rerun ellipticity and redshift notebooks with new catalog * fixed vardist nullgating * Remove old notebooks * Remove try-catch block from cached_dataset * Remove print statements from lensing_encoder * Pass flake8 check in catalog.py * attempted fixed error with nullgating --------- Co-authored-by: Yicun Duan <[email protected]> Co-authored-by: Yicun Duan <[email protected]> Co-authored-by: shreyasc <[email protected]>
- Loading branch information
4 people
authored
Aug 19, 2024
1 parent
bc88cfe
commit 41d09e1
Showing
26 changed files
with
3,205 additions
and
1,965 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
151 changes: 151 additions & 0 deletions
151
case_studies/weak_lensing/generate_dc2_lensing_catalog.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| import os | ||
| import pickle as pkl | ||
|
|
||
| import GCRCatalogs | ||
| import healpy as hp | ||
| import numpy as np | ||
| import pandas as pd | ||
| from GCRCatalogs import GCRQuery | ||
|
|
||
| GCRCatalogs.set_root_dir("/data/scratch/dc2_nfs/") | ||
|
|
||
| file_name = "dc2_lensing_catalog.pkl" | ||
| file_path = os.path.join("/data", "scratch", "dc2local", file_name) | ||
| file_already_populated = os.path.isfile(file_path) | ||
|
|
||
| if file_already_populated: | ||
| raise FileExistsError(f"{file_path} already exists.") | ||
|
|
||
|
|
||
| print("Loading object-with-truth-match...\n") # noqa: WPS421 | ||
|
|
||
| object_truth_cat = GCRCatalogs.load_catalog("desc_dc2_run2.2i_dr6_object_with_truth_match") | ||
|
|
||
| object_truth_df = object_truth_cat.get_quantities( | ||
| quantities=[ | ||
| "cosmodc2_id_truth", | ||
| "id_truth", | ||
| "objectId", | ||
| "match_objectId", | ||
| "truth_type", | ||
| "ra_truth", | ||
| "dec_truth", | ||
| "redshift_truth", | ||
| "flux_u_truth", | ||
| "flux_g_truth", | ||
| "flux_r_truth", | ||
| "flux_i_truth", | ||
| "flux_z_truth", | ||
| "flux_y_truth", | ||
| "mag_u_truth", | ||
| "mag_g_truth", | ||
| "mag_r_truth", | ||
| "mag_i_truth", | ||
| "mag_z_truth", | ||
| "mag_y_truth", | ||
| "Ixx_pixel", | ||
| "Iyy_pixel", | ||
| "Ixy_pixel", | ||
| "IxxPSF_pixel_u", | ||
| "IxxPSF_pixel_g", | ||
| "IxxPSF_pixel_r", | ||
| "IxxPSF_pixel_i", | ||
| "IxxPSF_pixel_z", | ||
| "IxxPSF_pixel_y", | ||
| "IyyPSF_pixel_u", | ||
| "IyyPSF_pixel_g", | ||
| "IyyPSF_pixel_r", | ||
| "IyyPSF_pixel_i", | ||
| "IyyPSF_pixel_z", | ||
| "IyyPSF_pixel_y", | ||
| "IxyPSF_pixel_u", | ||
| "IxyPSF_pixel_g", | ||
| "IxyPSF_pixel_r", | ||
| "IxyPSF_pixel_i", | ||
| "IxyPSF_pixel_z", | ||
| "IxyPSF_pixel_y", | ||
| "psf_fwhm_u", | ||
| "psf_fwhm_g", | ||
| "psf_fwhm_r", | ||
| "psf_fwhm_i", | ||
| "psf_fwhm_z", | ||
| "psf_fwhm_y", | ||
| ], | ||
| ) | ||
| object_truth_df = pd.DataFrame(object_truth_df) | ||
|
|
||
| max_ra = np.nanmax(object_truth_df["ra_truth"]) | ||
| min_ra = np.nanmin(object_truth_df["ra_truth"]) | ||
| max_dec = np.nanmax(object_truth_df["dec_truth"]) | ||
| min_dec = np.nanmin(object_truth_df["dec_truth"]) | ||
| ra_dec_filters = [f"ra >= {min_ra}", f"ra <= {max_ra}", f"dec >= {min_dec}", f"dec <= {max_dec}"] | ||
|
|
||
| vertices = hp.ang2vec( | ||
| np.array([min_ra, max_ra, max_ra, min_ra]), | ||
| np.array([min_dec, min_dec, max_dec, max_dec]), | ||
| lonlat=True, | ||
| ) | ||
| ipix = hp.query_polygon(32, vertices, inclusive=True) | ||
| healpix_filter = GCRQuery((lambda h: np.isin(h, ipix, assume_unique=True), "healpix_pixel")) | ||
|
|
||
| object_truth_df = object_truth_df[object_truth_df["truth_type"] == 1] | ||
|
|
||
|
|
||
| print("Loading CosmoDC2...\n") # noqa: WPS421 | ||
|
|
||
| config_overwrite = {"catalog_root_dir": "/data/scratch/dc2_nfs/cosmoDC2"} | ||
| cosmo_cat = GCRCatalogs.load_catalog("desc_cosmodc2", config_overwrite) | ||
|
|
||
| cosmo_df = cosmo_cat.get_quantities( | ||
| quantities=[ | ||
| "galaxy_id", | ||
| "ra", | ||
| "dec", | ||
| "ellipticity_1_true", | ||
| "ellipticity_2_true", | ||
| "shear_1", | ||
| "shear_2", | ||
| "convergence", | ||
| ], | ||
| filters=ra_dec_filters, | ||
| native_filters=healpix_filter, | ||
| ) | ||
| cosmo_df = pd.DataFrame(cosmo_df) | ||
|
|
||
|
|
||
| print("Merging...\n") # noqa: WPS421 | ||
|
|
||
| merge_df = object_truth_df.merge( | ||
| cosmo_df, left_on="cosmodc2_id_truth", right_on="galaxy_id", how="left" | ||
| ) | ||
|
|
||
| merge_df = merge_df[~merge_df["galaxy_id"].isna()] | ||
|
|
||
| merge_df.drop(columns=["ra_truth", "dec_truth"], inplace=True) | ||
|
|
||
| merge_df.rename( | ||
| columns={ | ||
| "redshift_truth": "redshift", | ||
| "flux_u_truth": "flux_u", | ||
| "flux_g_truth": "flux_g", | ||
| "flux_r_truth": "flux_r", | ||
| "flux_i_truth": "flux_i", | ||
| "flux_z_truth": "flux_z", | ||
| "flux_y_truth": "flux_y", | ||
| "mag_u_truth": "mag_u", | ||
| "mag_g_truth": "mag_g", | ||
| "mag_r_truth": "mag_r", | ||
| "mag_i_truth": "mag_i", | ||
| "mag_z_truth": "mag_z", | ||
| "mag_y_truth": "mag_y", | ||
| }, | ||
| inplace=True, | ||
| ) | ||
|
|
||
|
|
||
| print("Saving...\n") # noqa: WPS421 | ||
|
|
||
| with open(file_path, "wb") as f: | ||
| pkl.dump(merge_df, f) | ||
|
|
||
| print(f"Catalog has been saved at {file_path}") # noqa: WPS421 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.