diff --git a/.travis.yml b/.travis.yml index 9a77896..c53a5ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,8 @@ python: 2.7 install: - sudo apt-get update # Install the dependencies and the package: - - pip install -r requirements.txt - - python setup.py install +# - pip install -r requirements.txt +# - python setup.py install # Now get set up to run jupyter notebooks: - sudo apt-get install texlive-latex-recommended - sudo apt-get install texlive-latex-extra @@ -13,19 +13,19 @@ install: - sudo apt-get install texlive-publishers - sudo apt-get install chktex - sudo apt-get install dvipng - - pip install --upgrade jupyter - - pip install nbconvert +# - pip install --upgrade jupyter +# - pip install nbconvert # Finally get set up to build the docs - - pip install sphinx - - pip install sphinx_rtd_theme +# - pip install sphinx +# - pip install sphinx_rtd_theme script: # Run the unit tests: # - nosetests # Run the demo notebook: - - jupyter nbconvert --ExecutePreprocessor.kernel_name=python --ExecutePreprocessor.timeout=600 --to notebook --execute docs/notebooks/demo2.ipynb +# - jupyter nbconvert --ExecutePreprocessor.kernel_name=python --ExecutePreprocessor.timeout=600 --to notebook --execute docs/notebooks/demo2.ipynb # Build the docs, the same way readthedocs does it: - - cd docs ; sphinx-build -b html . _build/html ; cd - +# - cd docs ; sphinx-build -b html . _build/html ; cd - # Compile the paper - cd research/paper; make diff --git a/README.md b/README.md index 81992ef..65f5eea 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,35 @@ # chippr -Cosmological Hierarchical Inference with Probabilistic Photometric Redshifts. +Cosmological Hierarchical Inference with Probabilistic Photometric Redshifts ## Motivation -This repository is the home of `chippr`, a Python package for estimating quantities of cosmological interest from surveys of photometric redshift posterior probability distributions. It is a refactoring of my previous [work](https://github.com/aimalz/prob-z) on probabilistic photometric redshifts. +This repository is the home of `chippr`, a Python package for estimating quantities of cosmological interest from surveys of photometric redshift posterior probability distributions. +It is a refactoring of previous [work](https://github.com/aimalz/prob-z) on using probabilistic photometric redshifts to infer the redshift distribution. ## Examples -You can browse the demo notebook here in this repo: +You can browse the demo notebook here: * [Basic Demo for Python 2.7](http://htmlpreview.github.io/?https://github.com/aimalz/chippr/blob/master/docs/notebooks/demo2.html) ## Documentation -Documentation can be found on [ReadTheDocs](http://chippr.readthedocs.io/en/master/). The draft of the paper documenting the details of the method can be found [here](https://github.com/aimalz/chippr/blob/master/research/paper/draft.pdf). +Documentation can be found on [ReadTheDocs](http://chippr.readthedocs.io/en/master/). +The draft of the paper documenting the details of the method can be found [here](https://github.com/aimalz/chippr/blob/master/research/paper/draft.pdf). + +## Disclaimer + +As can be seen from the git history and Python version, this code is stale and should be understood to be a prototype, originally scoped out for applicability to SDSS DR10-era data of low dimensionality. +As a disclaimer, it will need a major upgrade for flexibility and computational scaling before it can run on data sets like those of modern and future galaxy surveys. ## People -* [Alex Malz](https://github.com/aimalz/qp/issues/new?body=@aimalz) (NYU) +* [Alex I. Malz](https://github.com/aimalz) (German Centre for Cosmological Lensing) ## License, Contributing etc -The code in this repo is available for re-use under the MIT license, which means that you can do whatever you like with it, just don't blame me. If you end up using any of the code or ideas you find here in your academic research, please cite me as `Malz et al, in preparation\footnote{\texttt{https://github.com/aimalz/chippr}}`. If you are interested in this project, please do drop me a line via the hyperlinked contact name above, or by [writing me an issue](https://github.com/aimalz/chippr/issues/new). To get started contributing to the `chippr` project, just fork the repo - pull requests are always welcome! +The code in this repo is available for re-use under the MIT license, which means that you can do whatever you like with it, just don't blame me. +If you end up using any of the code or ideas you find here in your academic research, please cite me as `Malz et al, in preparation\footnote{\texttt{https://github.com/aimalz/chippr}}`. +If you are interested in this project, please do drop me a line via the hyperlinked contact name above, or by [writing me an issue](https://github.com/aimalz/chippr/issues/new). +To get started contributing to the `chippr` project, just fork the repo -- pull requests are always welcome! diff --git a/chippr/catalog.py b/chippr/catalog.py index 06c605f..2dbda40 100644 --- a/chippr/catalog.py +++ b/chippr/catalog.py @@ -47,6 +47,7 @@ def __init__(self, params={}, vb=True, loc='.', prepend=''): if vb: print self.params + np.random.seed(d.seed) self.cat = {} self.dir = loc diff --git a/chippr/catalog_plots.py b/chippr/catalog_plots.py index f8b0ab4..83540c4 100644 --- a/chippr/catalog_plots.py +++ b/chippr/catalog_plots.py @@ -139,10 +139,10 @@ def plot_mega_scatter(zs, pfs, z_grid, grid_ends, truth=None, plot_loc='', prepe limval = (max(grid_ends) - min(grid_ends)) / (len(grid_ends) - 1.) scatplot.set_xlim([min(grid_ends)-limval, max(grid_ends)+limval]) scatplot.set_ylim([min(grid_ends)-limval, max(grid_ends)+limval]) - scatplot.set_xticks(np.linspace(min(grid_ends), np.ceil(max(grid_ends)), 5)) - scatplot.set_yticks(np.linspace(min(grid_ends), np.ceil(max(grid_ends)), 5)) - scatplot.set_xlabel(r'$z_{spec}$') - scatplot.set_ylabel(r'$z_{phot}$') + scatplot.set_xticks(np.linspace(np.floor(min(grid_ends)), np.ceil(max(grid_ends)), 5)) + scatplot.set_yticks(np.linspace(np.floor(min(grid_ends)), np.ceil(max(grid_ends)), 5)) + scatplot.set_xlabel(r'$z_{true}$') + scatplot.set_ylabel(r'$z_{est}$') scatplot.text(0.25, 3., r'mock $p(z\mid \mathrm{``data"})$', rotation=0, size=20) # scatplot.set_aspect(1.) diff --git a/chippr/log_z_dens_plots.py b/chippr/log_z_dens_plots.py index 86ef83e..93471f9 100644 --- a/chippr/log_z_dens_plots.py +++ b/chippr/log_z_dens_plots.py @@ -317,7 +317,7 @@ def plot_estimators(info, plot_dir, log=True, prepend='', metrics=True, mini=Tru f = plt.figure(figsize=(7.5, 7.5)) sps_log = f.add_subplot(1, 1, 1) sps_log.set_xlabel(r'$z$') - sps_log.set_xticks(np.linspace(min(info['bin_ends']), np.ceil(max(info['bin_ends'])), 5)) + sps_log.set_xticks(np.linspace(np.floor(min(info['bin_ends'])), np.ceil(max(info['bin_ends'])), 5)) # mini_sps.ticklabel_format(style='sci',axis='y') # tru, = @@ -344,7 +344,8 @@ def plot_estimators(info, plot_dir, log=True, prepend='', metrics=True, mini=Tru sps_log.set_ylim(-4., 1.) sps_log.set_ylabel(r'$\ln[n(z)]$') else: - sps_log.set_ylim(0., 3.) + sps_log.set_ylim(0., 4.) + sps_log.set_yticks([0,1,2,3,4]) sps_log.set_ylabel(r'$n(z)$') sps_log.set_xlim(info['bin_ends'][0], info['bin_ends'][-1]) @@ -501,7 +502,11 @@ def plot_estimators(info, plot_dir, log=True, prepend='', metrics=True, mini=Tru # sps_log.legend(handles=color_plots[:-1], fontsize='x-small', loc='lower center', frameon=False) sps_log.legend(fontsize='large', loc='upper right', frameon=False) - sps_log.text(0.25, -3.75, r'inferred $n(z)$', rotation=0, size=20) + if log: + sps_log.text(0.25, -3.75, r'inferred $n(z)$', rotation=0, size=20) + else: + sps_log.text(2., 0.75, r'inferred $n(z)$', rotation=0, size=20) + f.subplots_adjust(hspace=0, wspace=0) f.savefig(os.path.join(plot_dir, prepend+'estimators.png'), bbox_inches='tight', pad_inches = 0, dpi=d.dpi) print(info['stats']) diff --git a/research/paper/cleaner.sh b/research/paper/cleaner.sh index d84d72c..6275a3f 100644 --- a/research/paper/cleaner.sh +++ b/research/paper/cleaner.sh @@ -1,7 +1,7 @@ #infile='thesis.tex' #outfile='draft.tex' -sed -e "s/{\\textbackslash}'\\\\{i\\\\}/\\'{i}/g" thesis.bib | grep -E -v '^\W*(doi|url|urldate|abstract|file|keywords|annote|note)' > thesis.bib.tmp -mv thesis.bib.tmp thesis.bib +sed -e "s/{\\textbackslash}'\\\\{i\\\\}/\\'{i}/g" draft.bib | grep -E -v '^\W*(doi|url|urldate|abstract|file|keywords|annote|note)' > draft.bib.tmp +mv draft.bib.tmp draft.bib #grep -E -v '^(%|[[:blank:]]*%|\\COMMENT|\includegraphics*)' "$infile" | fold -w80 -s > "$outfile" diff --git a/research/paper/draft.bib b/research/paper/draft.bib index 294a051..c3426cc 100644 --- a/research/paper/draft.bib +++ b/research/paper/draft.bib @@ -4,7 +4,6 @@ @article{carrasco_kind_sparse_2014 volume = {441}, issn = {0035-8711}, shorttitle = {Sparse representation of photometric redshift probability density functions}, - doi = {10.1093/mnras/stu827}, number = {4}, journal = {Mon Not R Astron Soc}, author = {Carrasco Kind, Matias and Brunner, Robert J.}, @@ -17,7 +16,6 @@ @article{benitez_bayesian_2000 title = {Bayesian {Photometric} {Redshift} {Estimation}}, volume = {536}, issn = {0004-637X}, - doi = {10.1086/308947}, language = {en}, number = {2}, journal = {ApJ}, @@ -30,7 +28,6 @@ @article{krause_cosmolike_2017 title = {cosmolike {\textendash} cosmological likelihood analyses for photometric galaxy surveys}, volume = {470}, issn = {0035-8711}, - doi = {10.1093/mnras/stx1261}, number = {2}, journal = {Mon Not R Astron Soc}, author = {Krause, Elisabeth and Eifler, Tim}, @@ -47,11 +44,10 @@ @article{abell_lsst_2009 } @article{sadeh_annz2:_2016, - title = {{ANNz}2: {Photometric} {Redshift} and {Probability} {Distribution} {Function} {Estimation} using {Machine} {Learning}}, + title = {{ANNz2}: {Photometric} {Redshift} and {Probability} {Distribution} {Function} {Estimation} using {Machine} {Learning}}, volume = {128}, issn = {1538-3873}, - shorttitle = {{ANNz}2}, - doi = {10.1088/1538-3873/128/968/104502}, + shorttitle = {{ANNz2}}, language = {en}, number = {968}, journal = {PASP}, @@ -75,7 +71,6 @@ @article{hildebrandt_kids-450:_2017 volume = {465}, issn = {0035-8711}, shorttitle = {{KiDS}-450}, - doi = {10.1093/mnras/stw2805}, journal = {Mon Not R Astron Soc}, author = {Hildebrandt, H. and Viola, M. and Heymans, C. and Joudaki, S. and Kuijken, K. and Blake, C. and Erben, T. and Joachimi, B. and Klaes, D. and Miller, L. and Morrison, C. B. and Nakajima, R. and Verdoes Kleijn, G. and Amon, A. and Choi, A. and Covone, G. and de Jong, J. T. A. and Dvornik, A. and Fenech Conti, I. and Grado, A. and Harnois-D{\'e}raps, J. and Herbonnet, R. and Hoekstra, H. and K{\"o}hlinger, F. and McFarland, J. and Mead, A. and Merten, J. and Napolitano, N. and Peacock, J. A. and Radovich, M. and Schneider, P. and Simon, P. and Valentijn, E. A. and van den Busch, J. L. and van Uitert, E. and Van Waerbeke, L.}, month = feb, @@ -88,7 +83,6 @@ @article{herbel_redshift_2017 volume = {2017}, issn = {1475-7516}, shorttitle = {The redshift distribution of cosmological samples}, - doi = {10.1088/1475-7516/2017/08/035}, language = {en}, number = {08}, journal = {J. Cosmol. Astropart. Phys.}, @@ -99,8 +93,9 @@ @article{herbel_redshift_2017 @article{coe_fisher_2009, title = {Fisher {Matrices} and {Confidence} {Ellipses}: {A} {Quick}-{Start} {Guide} and {Software}}, + volume = {0906.4123}, shorttitle = {Fisher {Matrices} and {Confidence} {Ellipses}}, - journal = {arXiv:0906.4123 [astro-ph]}, + journal = {arXiv}, author = {Coe, Dan}, month = jun, year = {2009}, @@ -110,7 +105,6 @@ @article{hearin_general_2012 title = {General {Requirements} on {Matter} {Power} {Spectrum} {Predictions} for {Cosmology} with {Weak} {Lensing} {Tomography}}, volume = {2012}, issn = {1475-7516}, - doi = {10.1088/1475-7516/2012/04/034}, number = {04}, journal = {Journal of Cosmology and Astroparticle Physics}, author = {Hearin, Andrew P. and Zentner, Andrew R. and Ma, Zhaoming}, @@ -119,20 +113,10 @@ @article{hearin_general_2012 pages = {034--034}, } -@techreport{hoyle_dark_2017, - title = {Dark {Energy} {Survey} {Year} 1 {Results}: {Redshift} distributions of the weak lensing source galaxies}, - shorttitle = {Dark {Energy} {Survey} {Year} 1 {Results}}, - number = {Fermilab PUB-17-293-AE}, - author = {Hoyle, B. and Gruen, D. and Bernstein, G. M. and Rau, M. M. and De Vicente, J. and Hartley, W. G. and Gaztanaga, E. and DeRose, J. and Troxel, M. A. and Davis, C. and Alarcon, A. and MacCrann, N. and Prat, J. and S{\'a}nchez, C. and Sheldon, E. and Wechsler, R. H. and Asorey, J. and Becker, M. R. and Bonnett, C. and Rosell, A. Carnero and Carollo, D. and Kind, M. Carrasco and Castander, F. J. and Cawthon, R. and Chang, C. and Childress, M. and Davis, T. M. and Drlica-Wagner, A. and Gatti, M. and Glazebrook, K. and Gschwend, J. and Hinton, S. R. and Hoormann, J. K. and Kim, A. G. and King, A. and Kuehn, K. and Lewis, G. and Lidman, C. and Lin, H. and Macaulay, E. and Maia, M. A. G. and Martini, P. and Mudd, D. and M{\"o}ller, A. and Nichol, R. C. and Ogando, R. L. C. and Rollins, R. P. and Roodman, A. and Ross, A. J. and Rozo, E. and Rykoff, E. S. and Samuroff, S. and Sevilla-Noarbe, I. and Sharp, R. and Sommer, N. E. and Tucker, B. E. and Uddin, S. A. and Varga, T. N. and Vielzeuf, P. and Yuan, F. and Zhang, B. and Abbott, T. M. C. and Abdalla, F. B. and Allam, S. and Annis, J. and Bechtol, K. and Benoit-L{\'e}vy, A. and Bertin, E. and Brooks, D. and Buckley-Geer, E. and Burke, D. L. and Busha, M. T. and Capozzi, D. and Carretero, J. and Crocce, M. and D'Andrea, C. B. and da Costa, L. N. and DePoy, D. L. and Desai, S. and Diehl, H. T. and Doel, P. and Eifler, T. F. and Estrada, J. and Evrard, A. E. and Fernandez, E. and Flaugher, B. and Fosalba, P. and Frieman, J. and Garc{\'i}a-Bellido, J. and Gerdes, D. W. and Giannantonio, T. and Goldstein, D. A. and Gruendl, R. A. and Gutierrez, G. and Honscheid, K. and James, D. J. and Jarvis, M. and Jeltema, T. and Johnson, M. W. G. and Johnson, M. D. and Kirk, D. and Krause, E. and Kuhlmann, S. and Kuropatkin, N. and Lahav, O. and Li, T. S. and Lima, M. and March, M. and Marshall, J. L. and Melchior, P. and Menanteau, F. and Miquel, R. and Nord, B. and O'Neill, C. R. and Plazas, A. A. and Romer, A. K. and Sako, M. and Sanchez, E. and Santiago, B. and Scarpine, V. and Schindler, R. and Schubnell, M. and Smith, M. and Smith, R. C. and Soares-Santos, M. and Sobreira, F. and Suchyta, E. and Swanson, M. E. C. and Tarle, G. and Thomas, D. and Tucker, D. L. and Vikram, V. and Walker, A. R. and Weller, J. and Wester, W. and Wolf, R. C. and Yanny, B. and Zuntz, J.}, - month = aug, - year = {2017}, -} - @article{schmidt_improved_2013, title = {Improved photometric redshifts via enhanced estimates of system response, galaxy templates and magnitude priors}, volume = {431}, issn = {0035-8711}, - doi = {10.1093/mnras/stt373}, language = {en}, number = {3}, journal = {Mon Not R Astron Soc}, @@ -144,7 +128,8 @@ @article{schmidt_improved_2013 @article{mandelbaum_weak_2017, title = {Weak lensing for precision cosmology}, - journal = {arXiv:1710.03235 [astro-ph]}, + volume = {1710.03235}, + journal = {arXiv}, author = {Mandelbaum, Rachel}, month = oct, year = {2017}, @@ -154,7 +139,6 @@ @article{tanaka_photometric_2018 title = {Photometric redshifts for {Hyper} {Suprime}-{Cam} {Subaru} {Strategic} {Program} {Data} {Release} 1}, volume = {70}, issn = {0004-6264}, - doi = {10.1093/pasj/psx077}, language = {en}, number = {SP1}, journal = {Publ Astron Soc Jpn Nihon Tenmon Gakkai}, @@ -168,7 +152,6 @@ @article{asorey_galaxy_2016 title = {Galaxy clustering with photometric surveys using {PDF} redshift information}, volume = {459}, issn = {0035-8711, 1365-2966}, - doi = {10.1093/mnras/stw721}, number = {2}, journal = {Monthly Notices of the Royal Astronomical Society}, author = {Asorey, J. and Kind, M. Carrasco and Sevilla-Noarbe, I. and Brunner, R. J. and Thaler, J.}, @@ -178,10 +161,9 @@ @article{asorey_galaxy_2016 } @article{sheldon_photometric_2012, - title = {Photometric {Redshift} {Probability} {Distributions} for {Galaxies} in the {SDSS} {DR}8}, + title = {Photometric {Redshift} {Probability} {Distributions} for {Galaxies} in the {SDSS} {DR8}}, volume = {201}, issn = {0067-0049}, - doi = {10.1088/0067-0049/201/2/32}, language = {en}, number = {2}, journal = {ApJS}, @@ -190,20 +172,10 @@ @article{sheldon_photometric_2012 pages = {32}, } -@article{koo_photometric_1999, - title = {Photometric {Redshifts}: {A} {Perspective} from an {Old}-{Timer} on {Its} {Past}, {Present}, and {Potential}}, - shorttitle = {Photometric {Redshifts}}, - journal = {arXiv:astro-ph/9907273}, - author = {Koo, David C.}, - month = jul, - year = {1999}, -} - @article{mandelbaum_precision_2008, title = {Precision photometric redshift calibration for galaxy{\textendash}galaxy weak lensing}, volume = {386}, issn = {0035-8711}, - doi = {10.1111/j.1365-2966.2008.12947.x}, language = {en}, number = {2}, journal = {Mon Not R Astron Soc}, @@ -215,7 +187,7 @@ @article{mandelbaum_precision_2008 @article{laureijs_euclid_2011, title = {Euclid {Definition} {Study} {Report}}, - journal = {arXiv:1110.3193 [astro-ph]}, + journal = {ESA/SRE(2011)12}, author = {Laureijs, R. and Amiaux, J. and Arduini, S. and Augu{\`e}res, J.-L. and Brinchmann, J. and Cole, R. and Cropper, M. and Dabin, C. and Duvet, L. and Ealet, A. and Garilli, B. and Gondoin, P. and Guzzo, L. and Hoar, J. and Hoekstra, H. and Holmes, R. and Kitching, T. and Maciaszek, T. and Mellier, Y. and Pasian, F. and Percival, W. and Rhodes, J. and Criado, G. Saavedra and Sauvage, M. and Scaramella, R. and Valenziano, L. and Warren, S. and Bender, R. and Castander, F. and Cimatti, A. and F{\`e}vre, O. Le and Kurki-Suonio, H. and Levi, M. and Lilje, P. and Meylan, G. and Nichol, R. and Pedersen, K. and Popa, V. and Lopez, R. Rebolo and Rix, H.-W. and Rottgering, H. and Zeilinger, W. and Grupp, F. and Hudelot, P. and Massey, R. and Meneghetti, M. and Miller, L. and Paltani, S. and Paulin-Henriksson, S. and Pires, S. and Saxton, C. and Schrabback, T. and Seidel, G. and Walsh, J. and Aghanim, N. and Amendola, L. and Bartlett, J. and Baccigalupi, C. and Beaulieu, J.-P. and Benabed, K. and Cuby, J.-G. and Elbaz, D. and Fosalba, P. and Gavazzi, G. and Helmi, A. and Hook, I. and Irwin, M. and Kneib, J.-P. and Kunz, M. and Mannucci, F. and Moscardini, L. and Tao, C. and Teyssier, R. and Weller, J. and Zamorani, G. and Osorio, M. R. Zapatero and Boulade, O. and Foumond, J. J. and Di Giorgio, A. and Guttridge, P. and James, A. and Kemp, M. and Martignac, J. and Spencer, A. and Walton, D. and Bl{\"u}mchen, T. and Bonoli, C. and Bortoletto, F. and Cerna, C. and Corcione, L. and Fabron, C. and Jahnke, K. and Ligori, S. and Madrid, F. and Martin, L. and Morgante, G. and Pamplona, T. and Prieto, E. and Riva, M. and Toledo, R. and Trifoglio, M. and Zerbi, F. and Abdalla, F. and Douspis, M. and Grenet, C. and Borgani, S. and Bouwens, R. and Courbin, F. and Delouis, J.-M. and Dubath, P. and Fontana, A. and Frailis, M. and Grazian, A. and Koppenh{\"o}fer, J. and Mansutti, O. and Melchior, M. and Mignoli, M. and Mohr, J. and Neissner, C. and Noddle, K. and Poncet, M. and Scodeggio, M. and Serrano, S. and Shane, N. and Starck, J.-L. and Surace, C. and Taylor, A. and Verdoes-Kleijn, G. and Vuerli, C. and Williams, O. R. and Zacchei, A. and Altieri, B. and Sanz, I. Escudero and Kohley, R. and Oosterbroek, T. and Astier, P. and Bacon, D. and Bardelli, S. and Baugh, C. and Bellagamba, F. and Benoist, C. and Bianchi, D. and Biviano, A. and Branchini, E. and Carbone, C. and Cardone, V. and Clements, D. and Colombi, S. and Conselice, C. and Cresci, G. and Deacon, N. and Dunlop, J. and Fedeli, C. and Fontanot, F. and Franzetti, P. and Giocoli, C. and Garcia-Bellido, J. and Gow, J. and Heavens, A. and Hewett, P. and Heymans, C. and Holland, A. and Huang, Z. and Ilbert, O. and Joachimi, B. and Jennins, E. and Kerins, E. and Kiessling, A. and Kirk, D. and Kotak, R. and Krause, O. and Lahav, O. and van Leeuwen, F. and Lesgourgues, J. and Lombardi, M. and Magliocchetti, M. and Maguire, K. and Majerotto, E. and Maoli, R. and Marulli, F. and Maurogordato, S. and McCracken, H. and McLure, R. and Melchiorri, A. and Merson, A. and Moresco, M. and Nonino, M. and Norberg, P. and Peacock, J. and Pello, R. and Penny, M. and Pettorino, V. and Di Porto, C. and Pozzetti, L. and Quercellini, C. and Radovich, M. and Rassat, A. and Roche, N. and Ronayette, S. and Rossetti, E. and Sartoris, B. and Schneider, P. and Semboloni, E. and Serjeant, S. and Simpson, F. and Skordis, C. and Smadja, G. and Smartt, S. and Spano, P. and Spiro, S. and Sullivan, M. and Tilquin, A. and Trotta, R. and Verde, L. and Wang, Y. and Williger, G. and Zhao, G. and Zoubian, J. and Zucca, E.}, month = oct, year = {2011}, @@ -225,7 +197,6 @@ @article{malz_approximating_2018 title = {Approximating {Photo}- z {PDFs} for {Large} {Surveys}}, volume = {156}, issn = {1538-3881}, - doi = {10.3847/1538-3881/aac6b5}, language = {en}, number = {1}, journal = {AJ}, @@ -238,7 +209,6 @@ @article{leung_bayesian_2017 title = {Bayesian {Redshift} {Classification} of {Emission}-line {Galaxies} with {Photometric} {Equivalent} {Widths}}, volume = {843}, issn = {0004-637X}, - doi = {10.3847/1538-4357/aa71af}, language = {en}, number = {2}, journal = {ApJ}, @@ -252,7 +222,6 @@ @article{masters_mapping_2015 volume = {813}, issn = {0004-637X}, shorttitle = {Mapping the {Galaxy} {Color}{\textendash}{Redshift} {Relation}}, - doi = {10.1088/0004-637X/813/1/53}, language = {en}, number = {1}, journal = {ApJ}, @@ -265,10 +234,9 @@ @article{ball_robust_2008 title = {Robust {Machine} {Learning} {Applied} to {Astronomical} {Data} {Sets}. {III}. {Probabilistic} {Photometric} {Redshifts} for {Galaxies} and {Quasars} in the {SDSS} and {GALEX}}, volume = {683}, issn = {0004-637X}, - doi = {10.1086/589646}, language = {en}, number = {1}, - journal = {Astrophys. J.}, + journal = {The Astrophysical Journal}, author = {Ball, Nicholas M. and Brunner, Robert J. and Myers, Adam D. and Strand, Natalie E. and Alberts, Stacey L. and Tcheng, David}, month = aug, year = {2008}, @@ -276,20 +244,21 @@ @article{ball_robust_2008 } @article{baum_photoelectric_1962, + series = {Problems of {Extra}-{Galactic} {Research}}, title = {Photoelectric {Magnitudes} and {Red}-{Shifts}}, - number = {15}, - journal = {Proc. from IAU Symp.}, + volume = {15}, + journal = {Proceedings from IAU Symposium}, author = {Baum, W. A.}, - year = {1962} + year = {1962}, + pages = {390} } @article{bonnett_using_2015, title = {Using neural networks to estimate redshift distributions. {An} application to {CFHTLenS}}, volume = {449}, issn = {0035-8711}, - doi = {10.1093/mnras/stv230}, number = {1}, - journal = {Mon. Not. R. Astron. Soc.}, + journal = {Monthly Notices of the Royal Astronomical Society}, author = {Bonnett, C.}, month = mar, year = {2015}, @@ -300,10 +269,9 @@ @article{budavari_unified_2009 title = {A {UNIFIED} {FRAMEWORK} {FOR} {PHOTOMETRIC} {REDSHIFTS}}, volume = {695}, issn = {0004-637X}, - doi = {10.1088/0004-637X/695/1/747}, language = {en}, number = {1}, - journal = {Astrophys. J.}, + journal = {The Astrophysical Journal}, author = {Budav{\'a}ri, Tam{\'a}s}, month = apr, year = {2009}, @@ -314,10 +282,9 @@ @article{carliles_random_2010 title = {{RANDOM} {FORESTS} {FOR} {PHOTOMETRIC} {REDSHIFTS}}, volume = {712}, issn = {0004-637X}, - doi = {10.1088/0004-637X/712/1/511}, language = {en}, number = {1}, - journal = {Astrophys. J.}, + journal = {The Astrophysical Journal}, author = {Carliles, Samuel and Budav{\'a}ri, Tam{\'a}s and Heinis, S{\'e}bastien and Priebe, Carey and Szalay, Alexander S.}, month = mar, year = {2010}, @@ -328,36 +295,33 @@ @article{carrasco_kind_tpz:_2013 title = {{TPZ}: photometric redshift {PDFs} and ancillary information by using prediction trees and random forests}, volume = {432}, issn = {0035-8711}, - doi = {10.1093/mnras/stt574}, number = {2}, - journal = {Mon. Not. R. Astron. Soc.}, + journal = {Monthly Notices of the Royal Astronomical Society}, author = {Carrasco Kind, M. and Brunner, R. J.}, month = may, year = {2013}, - pages = {1483--1501} + pages = {1483--1501}, } @article{carrasco_kind_exhausting_2014, title = {Exhausting the information: novel {Bayesian} combination of photometric redshift {PDFs}}, volume = {442}, issn = {0035-8711}, - doi = {10.1093/mnras/stu1098}, number = {4}, - journal = {Mon. Not. R. Astron. Soc.}, + journal = {Monthly Notices of the Royal Astronomical Society}, author = {Carrasco Kind, M. and Brunner, R. J.}, month = jun, year = {2014}, - pages = {3380--3399} + pages = {3380--3399}, } @article{dipompeo_quasar_2015, title = {Quasar probabilities and redshifts from {WISE} mid-{IR} through {GALEX} {UV} photometry}, volume = {452}, issn = {0035-8711}, - doi = {10.1093/mnras/stv1562}, language = {en}, number = {3}, - journal = {Mon. Not. R. Astron. Soc.}, + journal = {Monthly Notices of the Royal Astronomical Society}, author = {DiPompeo, M. A. and Bovy, J. and Myers, A. D. and Lang, D.}, month = jul, year = {2015}, @@ -368,9 +332,8 @@ @article{foreman-mackey_emcee_2013 title = {emcee : {The} {MCMC} {Hammer}}, volume = {125}, issn = {00046280}, - doi = {10.1086/670067}, number = {925}, - journal = {Publ. Astron. Soc. Pacific}, + journal = {Publications of the Astronomical Society of the Pacific}, author = {Foreman-Mackey, Daniel and Hogg, David W. and Lang, Dustin and Goodman, Jonathan}, month = mar, year = {2013}, @@ -381,10 +344,9 @@ @article{foreman-mackey_exoplanet_2014 title = {{EXOPLANET} {POPULATION} {INFERENCE} {AND} {THE} {ABUNDANCE} {OF} {EARTH} {ANALOGS} {FROM} {NOISY}, {INCOMPLETE} {CATALOGS}}, volume = {795}, issn = {1538-4357}, - doi = {10.1088/0004-637X/795/1/64}, language = {en}, number = {1}, - journal = {Astrophys. J.}, + journal = {The Astrophysical Journal}, author = {Foreman-Mackey, Daniel and Hogg, David W. and Morton, Timothy D.}, month = oct, year = {2014}, @@ -395,8 +357,7 @@ @article{gorecki_new_2014 title = {A new method to improve photometric redshift reconstruction}, volume = {561}, issn = {0004-6361}, - doi = {10.1051/0004-6361/201321102}, - journal = {Astron. Astrophys.}, + journal = {Astronomy \& Astrophysics}, author = {Gorecki, Alexia and Abate, Alexandra and Ansari, R{\'e}za and Barrau, Aur{\'e}lien and Baumont, Sylvain and Moniez, Marc and Ricol, Jean-St{\'e}phane}, month = jan, year = {2014}, @@ -405,19 +366,29 @@ @article{gorecki_new_2014 @article{hogg_data_2012, title = {Data analysis recipes: {Probability} calculus for inference}, + volume = {1205.4446}, journal = {arXiv}, author = {Hogg, David W.}, month = may, - year = {2012} + year = {2012}, +} + +@incollection{koo_photometric_1999, + title = {Photometric {Redshifts}: {A} {Perspective} from an {Old}-{Timer} on {Its} {Past}, {Present}, and {Potential}}, + booktitle = {Photom. {Redshifts} {High} {Redshift} {Galaxies}}, + publisher = {ASP Conference Series}, + author = {Koo, David C.}, + editor = {Weymann, R and Storrie-Lombardi, L. and Sawicki, M. and Brunner, R.}, + month = jul, + year = {1999}, } @article{lima_estimating_2008, title = {Estimating the redshift distribution of photometric galaxy samples}, volume = {390}, issn = {00358711}, - doi = {10.1111/j.1365-2966.2008.13510.x}, number = {1}, - journal = {Mon. Not. R. Astron. Soc.}, + journal = {Monthly Notices of the Royal Astronomical Society}, author = {Lima, Marcos and Cunha, Carlos E. and Oyaizu, Hiroaki and Frieman, Joshua and Lin, Huan and Sheldon, Erin S.}, month = oct, year = {2008}, @@ -426,20 +397,19 @@ @article{lima_estimating_2008 @article{menard_clustering-based_2013, title = {Clustering-based redshift estimation: method and application to data}, + volume = {1303.4722}, journal = {arXiv}, author = {M{\'e}nard, Brice and Scranton, Ryan and Schmidt, Samuel and Morrison, Chris and Jeong, Donghui and Budavari, Tamas and Rahman, Mubdi}, month = mar, year = {2013}, - pages = {10} } @article{norberg_2df_2002, - title = {The 2dF {Galaxy} {Redshift} {Survey}: the {bJ}-band galaxy luminosity function and survey selection function}, + title = {The {2dF} {Galaxy} {Redshift} {Survey}: the {bJ}-band galaxy luminosity function and survey selection function}, volume = {336}, issn = {0035-8711}, - doi = {10.1046/j.1365-8711.2002.05831.x}, number = {3}, - journal = {Mon. Not. R. Astron. Soc.}, + journal = {Monthly Notices of the Royal Astronomical Society}, author = {Norberg, P. and Cole, S. and Baugh, C. M. and Frenk, C. S. and Baldry, I. and Bland-Hawthorn, J. and Bridges, T. and Cannon, R. and Colless, M. and Collins, C. and Couch, W. and Cross, N. J. G. and Dalton, G. and De Propris, R. and Driver, S. P. and Efstathiou, G. and Ellis, R. S. and Glazebrook, K. and Jackson, C. and Lahav, O. and Lewis, I. and Lumsden, S. and Maddox, S. and Madgwick, D. and Peacock, J. A. and Peterson, B. A. and Sutherland, W. and Taylor, K.}, month = nov, year = {2002}, @@ -450,7 +420,6 @@ @article{sanchez_clustering_2013 title = {The clustering of galaxies in the {SDSS}-{III} {Baryon} {Oscillation} {Spectroscopic} {Survey}: cosmological constraints from the full shape of the clustering wedges}, volume = {433}, issn = {0035-8711}, - doi = {10.1093/mnras/stt799}, number = {2}, journal = {Mon. Not. R. Astron. Soc.}, author = {Sanchez, A. G. and Kazin, E. A. and Beutler, F. and Chuang, C.-H. and Cuesta, A. J. and Eisenstein, D. J. and Manera, M. and Montesano, F. and Nichol, R. C. and Padmanabhan, N. and Percival, W. and Prada, F. and Ross, A. J. and Schlegel, D. J. and Tinker, J. and Tojeiro, R. and Weinberg, D. H. and Xu, X. and Brinkmann, J. and Brownstein, J. R. and Schneider, D. P. and Thomas, D.}, @@ -460,12 +429,11 @@ @article{sanchez_clustering_2013 } @article{van_breukelen_reliable_2009, - title = {A reliable cluster detection technique using photometric redshifts: introducing the 2TecX algorithm}, + title = {A reliable cluster detection technique using photometric redshifts: introducing the {2TecX} algorithm}, volume = {395}, issn = {00358711}, - doi = {10.1111/j.1365-2966.2009.14692.x}, number = {4}, - journal = {Mon. Not. R. Astron. Soc.}, + journal = {Monthly Notices of the Royal Astronomical Society}, author = {van Breukelen, Caroline and Clewley, Lee}, month = jun, year = {2009}, @@ -476,42 +444,38 @@ @article{viironen_high_2015 title = {High redshift galaxies in the {ALHAMBRA} survey}, volume = {576}, issn = {0004-6361}, - doi = {10.1051/0004-6361/201425382}, language = {en}, - journal = {Astron. Astrophys.}, + journal = {Astronomy \& Astrophysics}, author = {Viironen, K. and Mar{\'i}n-Franch, A. and L{\'o}pez-Sanjuan, C. and Varela, J. and Chaves-Montero, J. and Crist{\'o}bal-Hornillos, D. and Molino, A. and Fern{\'a}ndez-Soto, A. and Vilella-Rojo, G. and Ascaso, B. and Cenarro, A. J. and Cervi{\~n}o, M. and Cepa, J. and Ederoclite, A. and M{\'a}rquez, I. and Masegosa, J. and Moles, M. and Oteo, I. and Povi{\'c}, M. and Aguerri, J. A. L. and Alfaro, E. and Aparicio-Villegas, T. and Ben{\'i}tez, N. and Broadhurst, T. and Cabrera-Ca{\~n}o, J. and Castander, J. F. and Del Olmo, A. and Gonz{\'a}lez Delgado, R. M. and Husillos, C. and Infante, L. and Mart{\'i}nez, V. J. and Perea, J. and Prada, F. and Quintana, J. M.}, month = mar, year = {2015}, pages = {A25} } -@article{dahlen_critical_2013, - title = {A {Critical} {Assessment} of {Photometric} {Redshift} {Methods}: {A} {CANDELS} {Investigation}}, - volume = {775}, - doi = {10.1088/0004-637X/775/2/93}, - journal = {{\textbackslash}apj}, - author = {Dahlen, T. and Mobasher, B. and Faber, S. M. and Ferguson, H. C. and Barro, G. and Finkelstein, S. L. and Finlator, K. and Fontana, A. and Gruetzbauch, R. and Johnson, S. and Pforr, J. and Salvato, M. and Wiklind, T. and Wuyts, S. and Acquaviva, V. and Dickinson, M. E. and Guo, Y. and Huang, J. and Huang, K.-H. and Newman, J. A. and Bell, E. F. and Conselice, C. J. and Galametz, A. and Gawiser, E. and Giavalisco, M. and Grogin, N. A. and Hathi, N. and Kocevski, D. and Koekemoer, A. M. and Koo, D. C. and Lee, K.-S. and McGrath, E. J. and Papovich, C. and Peth, M. and Ryan, R. and Somerville, R. and Weiner, B. and Wilson, G.}, - month = oct, - year = {2013}, - pages = {93} -} - @article{hildebrandt_phat:_2010, title = {{PHAT}: {PHoto}-z {Accuracy} {Testing}}, volume = {523}, - doi = {10.1051/0004-6361/201014885}, - journal = {{\textbackslash}aap}, + journal = {Astronomy \& Astrophysics}, author = {Hildebrandt, H. and Arnouts, S. and Capak, P. and Moustakas, L. A. and Wolf, C. and Abdalla, F. B. and Assef, R. J. and Banerji, M. and Ben{\'i}tez, N. and Brammer, G. B. and Budav{\'a}ri, T. and Carliles, S. and Coe, D. and Dahlen, T. and Feldmann, R. and Gerdes, D. and Gillis, B. and Ilbert, O. and Kotulla, R. and Lahav, O. and Li, I. H. and Miralles, J.-M. and Purger, N. and Schmidt, S. and Singal, J.}, month = nov, year = {2010}, pages = {A31} } +@article{leistedt_data-driven_2017, + title = {Data-driven, {Interpretable} {Photometric} {Redshifts} {Trained} on {Heterogeneous} and {Unrepresentative} {Data}}, + volume = {838}, + journal = {The Astrophysical Journal}, + author = {Leistedt, B. and Hogg, D. W.}, + month = mar, + year = {2017}, + pages = {5}, +} + @article{leistedt_hierarchical_2016, title = {Hierarchical {Bayesian} inference of galaxy redshift distributions from photometric surveys}, volume = {460}, issn = {0035-8711}, - doi = {10.1093/mnras/stw1304}, language = {en}, number = {4}, journal = {Mon Not R Astron Soc}, @@ -523,8 +487,9 @@ @article{leistedt_hierarchical_2016 @article{jain_whole_2015, title = {The {Whole} is {Greater} than the {Sum} of the {Parts}: {Optimizing} the {Joint} {Science} {Return} from {LSST}, {Euclid} and {WFIRST}}, + volume = {1501.07897}, shorttitle = {The {Whole} is {Greater} than the {Sum} of the {Parts}}, - journal = {arXiv:1501.07897 [astro-ph]}, + journal = {arXiv}, author = {Jain, B. and Spergel, D. and Bean, R. and Connolly, A. and Dell'antonio, I. and Frieman, J. and Gawiser, E. and Gehrels, N. and Gladney, L. and Heitmann, K. and Helou, G. and Hirata, C. and Ho, S. and Ivezi{\'c}, {\v Z} and Jarvis, M. and Kahn, S. and Kalirai, J. and Kim, A. and Lupton, R. and Mandelbaum, R. and Marshall, P. and Newman, J. A. and Perlmutter, S. and Postman, M. and Rhodes, J. and Strauss, M. A. and Tyson, J. A. and Walkowicz, L. and Wood-Vasey, W. M.}, month = jan, year = {2015}, @@ -537,24 +502,10 @@ @misc{rohatgi_webplotdigitizer_2019 year = {2019}, } -@misc{schmidt_evaluation_nodate, - title = {Evaluation of probabilistic photometric redshift estimation approaches for {LSST}}, - author = {Schmidt, Sam and Malz, Alex I.} -} - -@article{leistedt_hierarchical_2018, - title = {Hierarchical modeling and statistical calibration for photometric redshifts}, - journal = {arXiv:1807.01391 [astro-ph]}, - author = {Leistedt, Boris and Hogg, David W. and Wechsler, Risa H. and DeRose, Joe}, - month = jul, - year = {2018}, -} - @article{yang_calibrating_2018, title = {Calibrating magnification bias for the {EG} statistic to test general relativity}, volume = {481}, issn = {0035-8711}, - doi = {10.1093/mnras/sty2353}, language = {en}, number = {2}, journal = {Mon Not R Astron Soc}, @@ -568,7 +519,6 @@ @article{abruzzo_impact_2019 title = {The impact of photometric redshift errors on lensing statistics in ray-tracing simulations}, volume = {486}, issn = {0035-8711}, - doi = {10.1093/mnras/stz1016}, language = {en}, number = {2}, journal = {Mon Not R Astron Soc}, @@ -583,7 +533,6 @@ @article{hildebrandt_cfhtlens:_2012 volume = {421}, issn = {0035-8711}, shorttitle = {{CFHTLenS}}, - doi = {10.1111/j.1365-2966.2012.20468.x}, language = {en}, number = {3}, journal = {Mon Not R Astron Soc}, @@ -598,7 +547,6 @@ @article{benjamin_cfhtlens_2013 volume = {431}, issn = {0035-8711}, shorttitle = {{CFHTLenS} tomographic weak lensing}, - doi = {10.1093/mnras/stt276}, language = {en}, number = {2}, journal = {Mon Not R Astron Soc}, @@ -612,7 +560,6 @@ @article{kelly_weighing_2014 title = {Weighing the {Giants} {\textendash} {II}. {Improved} calibration of photometry from stellar colours and accurate photometric redshifts}, volume = {439}, issn = {0035-8711}, - doi = {10.1093/mnras/stt1946}, language = {en}, number = {1}, journal = {Mon Not R Astron Soc}, @@ -621,3 +568,98 @@ @article{kelly_weighing_2014 year = {2014}, pages = {28--47}, } + +@article{ma_effects_2006, + title = {Effects of {Photometric} {Redshift} {Uncertainties} on {Weak}-{Lensing} {Tomography}}, + volume = {636}, + issn = {0004-637X}, + language = {en}, + number = {1}, + journal = {ApJ}, + author = {Ma, Zhaoming and Hu, Wayne and Huterer, Dragan}, + month = jan, + year = {2006}, + pages = {21}, +} + +@article{leistedt_hierarchical_2019, + title = {Hierarchical {Modeling} and {Statistical} {Calibration} for {Photometric} {Redshifts}}, + volume = {881}, + issn = {0004-637X}, + language = {en}, + number = {1}, + journal = {ApJ}, + author = {Leistedt, Boris and Hogg, David W. and Wechsler, Risa H. and DeRose, Joe}, + month = aug, + year = {2019}, + pages = {80}, +} + +@article{awan_angular_2019, + title = {Angular {Correlation} {Function} {Estimators} {Accounting} for {Contamination} from {Probabilistic} {Distance} {Measurements}}, + volume = {1911.07832}, + journal = {arXiv}, + author = {Awan, Humna and Gawiser, Eric}, + month = dec, + year = {2019}, +} + +@article{hoyle_dark_2018, + title = {Dark {Energy} {Survey} {Year} 1 {Results}: redshift distributions of the weak-lensing source galaxies}, + volume = {478}, + issn = {0035-8711}, + shorttitle = {Dark {Energy} {Survey} {Year} 1 {Results}}, + language = {en}, + number = {1}, + journal = {Mon Not R Astron Soc}, + author = {Hoyle, B. and Gruen, D. and Bernstein, G. M. and Rau, M. M. and De Vicente, J. and Hartley, W. G. and Gaztanaga, E. and DeRose, J. and Troxel, M. A. and Davis, C. and Alarcon, A. and MacCrann, N. and Prat, J. and S{\'a}nchez, C. and Sheldon, E. and Wechsler, R. H. and Asorey, J. and Becker, M. R. and Bonnett, C. and Rosell, A. Carnero and Carollo, D. and Kind, M. Carrasco and Castander, F. J. and Cawthon, R. and Chang, C. and Childress, M. and Davis, T. M. and Drlica-Wagner, A. and Gatti, M. and Glazebrook, K. and Gschwend, J. and Hinton, S. R. and Hoormann, J. K. and Kim, A. G. and King, A. and Kuehn, K. and Lewis, G. and Lidman, C. and Lin, H. and Macaulay, E. and Maia, M. a. G. and Martini, P. and Mudd, D. and M{\"o}ller, A. and Nichol, R. C. and Ogando, R. L. C. and Rollins, R. P. and Roodman, A. and Ross, A. J. and Rozo, E. and Rykoff, E. S. and Samuroff, S. and Sevilla-Noarbe, I. and Sharp, R. and Sommer, N. E. and Tucker, B. E. and Uddin, S. A. and Varga, T. N. and Vielzeuf, P. and Yuan, F. and Zhang, B. and Abbott, T. M. C. and Abdalla, F. B. and Allam, S. and Annis, J. and Bechtol, K. and Benoit-L{\'e}vy, A. and Bertin, E. and Brooks, D. and Buckley-Geer, E. and Burke, D. L. and Busha, M. T. and Capozzi, D. and Carretero, J. and Crocce, M. and D{\textquoteright}Andrea, C. B. and da Costa, L. N. and DePoy, D. L. and Desai, S. and Diehl, H. T. and Doel, P. and Eifler, T. F. and Estrada, J. and Evrard, A. E. and Fernandez, E. and Flaugher, B. and Fosalba, P. and Frieman, J. and Garc{\'i}a-Bellido, J. and Gerdes, D. W. and Giannantonio, T. and Goldstein, D. A. and Gruendl, R. A. and Gutierrez, G. and Honscheid, K. and James, D. J. and Jarvis, M. and Jeltema, T. and Johnson, M. W. G. and Johnson, M. D. and Kirk, D. and Krause, E. and Kuhlmann, S. and Kuropatkin, N. and Lahav, O. and Li, T. S. and Lima, M. and March, M. and Marshall, J. L. and Melchior, P. and Menanteau, F. and Miquel, R. and Nord, B. and O{\textquoteright}Neill, C. R. and Plazas, A. A. and Romer, A. K. and Sako, M. and Sanchez, E. and Santiago, B. and Scarpine, V. and Schindler, R. and Schubnell, M. and Smith, M. and Smith, R. C. and Soares-Santos, M. and Sobreira, F. and Suchyta, E. and Swanson, M. E. C. and Tarle, G. and Thomas, D. and Tucker, D. L. and Vikram, V. and Walker, A. R. and Weller, J. and Wester, W. and Wolf, R. C. and Yanny, B. and Zuntz, J.}, + month = jul, + year = {2018}, + pages = {592--610}, +} + +@article{dahlen_critical_2013, + title = {A {Critical} {Assessment} of {Photometric} {Redshift} {Methods}: {A} {CANDELS} {Investigation}}, + volume = {775}, + journal = {The Astrophysical Journal}, + author = {Dahlen, T. and Mobasher, B. and Faber, S. M. and Ferguson, H. C. and Barro, G. and Finkelstein, S. L. and Finlator, K. and Fontana, A. and Gruetzbauch, R. and Johnson, S. and Pforr, J. and Salvato, M. and Wiklind, T. and Wuyts, S. and Acquaviva, V. and Dickinson, M. E. and Guo, Y. and Huang, J. and Huang, K.-H. and Newman, J. A. and Bell, E. F. and Conselice, C. J. and Galametz, A. and Gawiser, E. and Giavalisco, M. and Grogin, N. A. and Hathi, N. and Kocevski, D. and Koekemoer, A. M. and Koo, D. C. and Lee, K.-S. and McGrath, E. J. and Papovich, C. and Peth, M. and Ryan, R. and Somerville, R. and Weiner, B. and Wilson, G.}, + month = oct, + year = {2013}, + pages = {93} +} + +@article{schmidt_evaluation_2020, + title = {Evaluation of probabilistic photometric redshift estimation approaches for {LSST}}, + volume = {2001.03621}, + journal = {arXiv}, + author = {Schmidt, S. J. and Malz, A. I. and Soo, J. Y. H. and Almosallam, I. A. and Brescia, M. and Cavuoti, S. and Cohen-Tanugi, J. and Connolly, A. J. and DeRose, J. and Freeman, P. E. and Graham, M. L. and Iyer, K. G. and Jarvis, M. J. and Kalmbach, J. B. and Kovacs, E. and Lee, A. B. and Longo, G. and Morrison, C. B. and Newman, J. A. and Nourbakhsh, E. and Nuss, E. and Pospisil, T. and Tranin, H. and Wechsler, R. H. and Zhou, R. and Izbicki, R. and Collaboration, The LSST Dark Energy Science}, + month = jan, + year = {2020}, +} + +@article{hogg_inferring_2010, + title = {{INFERRING} {THE} {ECCENTRICITY} {DISTRIBUTION}}, + volume = {725}, + issn = {0004-637X}, + language = {en}, + number = {2}, + journal = {ApJ}, + author = {Hogg, David W. and Myers, Adam D. and Bovy, Jo}, + month = dec, + year = {2010}, + pages = {2166--2175}, +} + +@article{gelman_inference_1992, + title = {Inference from {Iterative} {Simulation} {Using} {Multiple} {Sequences}}, + volume = {7}, + issn = {0883-4237, 2168-8745}, + language = {EN}, + number = {4}, + journal = {Statist. Sci.}, + author = {Gelman, Andrew and Rubin, Donald B.}, + month = nov, + year = {1992}, + zmnumber = {06853057}, + pages = {457--472}, +} diff --git a/research/paper/draft.pdf b/research/paper/draft.pdf index 8611a0b..a5c3126 100644 Binary files a/research/paper/draft.pdf and b/research/paper/draft.pdf differ diff --git a/research/paper/draft.tex b/research/paper/draft.tex index d07f2e8..49372d1 100644 --- a/research/paper/draft.tex +++ b/research/paper/draft.tex @@ -8,14 +8,17 @@ \usepackage{hyperref} %\usepackage{graphicx} %\usepackage{lineno} +\usepackage[percent]{overpic} +\usepackage{float} +\usepackage{wrapfig} -\usetikzlibrary{shapes.geometric, arrows} -\usetikzlibrary{fit} - -\tikzstyle{hyper} = [circle, text centered, draw=black] -\tikzstyle{param} = [circle, text centered, draw=black] -\tikzstyle{data} = [circle, text centered, draw=black, line width=2pt] -\tikzstyle{arrow} = [thick,->,>=stealth] +%\usetikzlibrary{shapes.geometric, arrows} +%\usetikzlibrary{fit} +% +%\tikzstyle{hyper} = [circle, text centered, draw=black] +%\tikzstyle{param} = [circle, text centered, draw=black] +%\tikzstyle{data} = [circle, text centered, draw=black, line width=2pt] +%\tikzstyle{arrow} = [thick,->,>=stealth] %\usepackage{soul} \usepackage[title]{appendix} @@ -51,7 +54,7 @@ \newcommand{\data}{\ensuremath{\vec{d}}}% could change to bold \newcommand{\like}{\mathscr{L}} -\newcommand{\pr}[1]{\ensuremath{\mathrm{p}(#1)}}% could change to Prob or Pr +\newcommand{\pr}[1]{\ensuremath{\mathrm{p}(#1)}}% could change to Prob or Pr \newcommand{\expect}[1]{\left<#1\right>} \newcommand{\normal}[2]{\mathcal{N} (#1, #2)} \newcommand{\gvn}{\mid}% could use | or \vert @@ -90,11 +93,13 @@ \begin{abstract} A trustworthy estimate of the redshift distribution $n(z)$ is crucial for using weak gravitational lensing and large-scale structure of galaxy catalogs to study cosmology. -\dwh{Spectroscopic redshifts for the dim and numerous galaxies of next-generation weak-lensing surveys are expected to be unavailable, making photometric redshift (photo-$z$) probability density functions (PDFs) the next-best alternative for comprehensively encapsulating the nontrivial systematics affecting photo-$z$ point estimation. -The established stacked estimator of $n(z)$ avoids reducing photo-$z$ PDFs to point estimates but yields a systematically biased estimate of $n(z)$ that worsens with decreasing signal-to-noise, the very regime where photo-$z$ PDFs are most necessary.} +Spectroscopic redshifts for the dim and numerous galaxies of next-generation weak-lensing surveys are expected to be unavailable, making photometric redshift (photo-$z$) probability density functions (PDFs) the next-best alternative for comprehensively encapsulating the nontrivial systematics affecting photo-$z$ point estimation. +The established stacked estimator of $n(z)$ avoids reducing photo-$z$ PDFs to point estimates but yields a systematically biased estimate of $n(z)$ that worsens with decreasing signal-to-noise, the very regime where photo-$z$ PDFs are most necessary. We introduce Cosmological Hierarchical Inference with Probabilistic Photometric Redshifts (\textsc{CHIPPR}), a statistically rigorous probabilistic graphical model of redshift-dependent photometry, which correctly propagates the redshift uncertainty information beyond the best-fit estimator of $n(z)$ produced by traditional procedures and is provably the only self-consistent way to recover $n(z)$ from photo-$z$ PDFs. -We present the \texttt{chippr} prototype code and use it to forecast constraints in the space of cosmological parameters, noting, however, that the mathematically justifiable approach incurs computational expense. -The \textsc{CHIPPR} approach is applicable to any one-point statistic of any random variable, provided the prior probability density used to produce the posteriors is explicitly known; +We present the \texttt{chippr} prototype code +%and use it to forecast constraints in the space of cosmological parameters +, noting that the mathematically justifiable approach incurs computational expense. +The \textsc{CHIPPR} approach is applicable to any one-point statistic of any random variable, provided the prior probability density used to produce the posteriors is explicitly known; if the prior is implicit, as may be the case for popular photo-$z$ techniques, then the resulting posterior PDFs cannot be used for scientific inference. We therefore recommend that the photo-$z$ community focus on developing methodologies that enable the recovery of photo-$z$ likelihoods with support over all redshifts, either directly or via a known prior probability density. \end{abstract} @@ -103,44 +108,46 @@ \maketitle +%\aim{TODO: Add test case panel labels to all figs of \chippr\ sims/outputs. Take higher-z samples in mock \pzpdf\ plots relative to outlier populations. Include $\bar{z}$ in results plots.} + \section{Introduction} \label{sec:intro} % what are photo-zs? -Photometric redshift (\pz) estimation has been a staple of studies of galaxy evolution, large-scale structure, and cosmology since its conception half a century ago \citep{baum_photoelectric_1962}. -An extremely coarse spectrum in the form of photometry in a handful of broadband filters can be an effective substitute for the time- and photon-intensive process of obtaining a spectroscopic redshift (\sz), a procedure that may only be applied to relatively bright galaxies. +Photometric redshift (\pz) estimation has been a staple of studies of galaxy evolution, large-scale structure, and cosmology since its conception half a century ago \citep{baum_photoelectric_1962}. +An extremely coarse spectrum in the form of photometry in a handful of broadband filters can be an effective substitute for the time- and photon-intensive process of obtaining a spectroscopic redshift (\sz), a procedure that may only be applied to relatively bright galaxies. Once the photometric colors are calibrated against either a library of spectral energy distribution (SED) templates or a data set of spectra for galaxies with known redshifts, a correspondence between photometric colors and redshifts may be constructed, forming a trustworthy basis for \pz\ estimation or testing. % why do we need photo-zs? -Calculations of correlation functions of cosmic shears and galaxy positions that constrain the cosmological parameters require large numbers of high-confidence redshifts of surveyed galaxies. -Many more \pz s may be obtained in the time it would take to observe a smaller number of \sz s, and \pz s may be measured for galaxies too dim for accurate \sz\ confirmation, permitting the compilation of large catalogs of galaxies spanning a broad range of redshifts and luminosities. -\Pz s have thus enabled the era of precision cosmology, heralded by weak gravitational lensing tomography and baryon acoustic oscillation peak measurements. +Calculations of correlation functions of cosmic shears and galaxy positions that constrain the cosmological parameters require large numbers of high-confidence redshifts of surveyed galaxies. +Many more \pz s may be obtained in the time it would take to observe a smaller number of \sz s, and \pz s may be measured for galaxies too dim for accurate \sz\ confirmation, permitting the compilation of large catalogs of galaxies spanning a broad range of redshifts and luminosities. +\Pz s have thus enabled the era of precision cosmology, heralded by weak gravitational lensing tomography and baryon acoustic oscillation peak measurements. % what's wrong with photo-zs? -However, \pz s are susceptible to inaccuracy and imprecision in the form of their inherent noisiness resulting from the coarseness of photometric filters, catastrophic errors in which galaxies of one SED at one redshift are mistaken for galaxies of another SED at a different redshift, and systematics introduced by observational techniques, data reduction processes, and training or template set limitations. -Figure~\ref{fig:pedagogical_scatter} illustrates the relationship between photometry and redshift generically by showing ``data'' in one dimension for visualization purposes, suggesting that a special nonlinear projection of the photometry could more-or-less yield a one-to-one relationship with true redshifts. -The similarity to traditional $z_{\mathrm{spec}}$ versus $z_{\mathrm{phot}}$ plots of \pz\ point estimates is no coincidence, as \pz\ point estimates effectively are indeed one such special nonlinear projection of the data. +However, \pz s are susceptible to inaccuracy and imprecision in the form of their inherent noisiness resulting from the coarseness of photometric filters, catastrophic errors in which galaxies of one SED at one redshift are mistaken for galaxies of another SED at a different redshift, and systematics introduced by observational techniques, data reduction processes, and training or template set limitations. +Figure~\ref{fig:pedagogical_scatter} is an adaptation of the ubiquitous plots of \pz\ vs. \sz\ illustrating the assumptions underlying \pz\ estimation in general, that \sz s are a good approximation to true redshifts and \pz s represent special non-linear projections of observed photometry to a scalar variable that approximates the true redshift. \begin{figure} \begin{center} \includegraphics[width=0.45\textwidth]{figures/jain05.png} \caption{ - A generic probability space (darker in areas of higher probability density) of redshift ($x$-axis) and data ($y$-axis), where the data is projected into a single dimension, with vertical cuts and marginals (cyan) indicating the construction of likelihoods and horizontal cuts and marginals (magenta) indicating the construction of posteriors. - The data (black points) used to generate the contours were extracted from \citet{jain_whole_2015} using WebPlotDigitizer \citep{rohatgi_webplotdigitizer_2019}, with the ideal redshift estimation provided for reference (red diagonal). + A generic probability space (darker in areas of higher probability density) of true redshift ($x$-axis) and a nonlinear projection of photometric data ($y$-axis), with vertical cuts and marginals (orange) indicating the construction of likelihoods and horizontal cuts and marginals (blue) indicating the construction of posteriors, with a theoretically perfect \pz\ estimate on the diagonal (yellow) for reference. + The data points were extracted using WebPlotDigitizer \citep{rohatgi_webplotdigitizer_2019} from a \sz\ vs. \pz\ plot in \citet{jain_whole_2015}. +% \aim{TODO: recolor insets/bars to not be same as gradient.} } \label{fig:pedagogical_scatter} \end{center} \end{figure} % how do we interpret photo-z imperfections? -There are several varieties of generally non-Gaussian deviation from a \dwh{trivial relationship} between redshift and data in Figure~\ref{fig:pedagogical_scatter}, represented by a $y = x$ diagonal line. +There are several varieties of generally non-Gaussian deviation from a trivial relationship between redshift and data in Figure~\ref{fig:pedagogical_scatter}, represented by a $y = x$ diagonal line. The coarseness of the photometric filters causes scatter about the diagonal, with larger scatter perpendicular to the diagonal at redshifts where highly identifiable spectral features pass between the filters, as well as higher scatter at high redshifts where faint galaxies with large photometric errors are more abundant. There are populations of outliers, far from the diagonal, comprised of galaxies for which the redshift estimate is catastrophically distinct from the true redshift, showing that outliers are not uniformly distributed nor restricted to long tails away from a Gaussian scatter. And, though hardly perceptible in the plot, there is a systematic bias, wherein the average of the points would not lie on the diagonal but would be offset by a small bias, suggested by the trend of high-redshift points to lie below the diagonal. % how much do these imperfections matter? Once propagated through the calculations of correlation functions of cosmic shear and galaxy positions, \pz\ errors are a dominant contributor to the total uncertainties reported on cosmological parameters \citep{abruzzo_impact_2019}. -As progress has been made on the influence of other sources of systematic error, the uncertainties associated with \pz s have come to dominate the error budget of cosmological parameter estimates made by current surveys such as \des\ \citep{hoyle_dark_2017}, \project{HSC} \citep{tanaka_photometric_2018}, and \project{KiDS} \citep{hildebrandt_kids-450:_2017}. +As progress has been made on the influence of other sources of systematic error, the uncertainties associated with \pz s have come to dominate the error budget of cosmological parameter estimates made by current surveys such as \des\ \citep{hoyle_dark_2018}, \project{HSC} \citep{tanaka_photometric_2018}, and \project{KiDS} \citep{hildebrandt_kids-450:_2017}. Based on the goals of a photometric galaxy survey, limits can be placed on the tolerance to these effects. For example, the Science Requirements Document \citep{mandelbaum_weak_2017} states \lsst's requirements for the main cosmological sample, reproduced in Table~\ref{tab:lsstsrd}. @@ -152,7 +159,7 @@ \section{Introduction} Number of galaxies & $\approx 10^{7}$\\ Root-mean-square error & $< 0.02 (1 + z)$\\ $3 \sigma$ catastrophic outlier rate & $< 10\%$\\ - Bias & $< 0.003 (1 + z)$\\ + Canonical bias & $< 0.003 (1 + z)$\\ \end{tabular} \label{tab:lsstsrd} \end{center} @@ -163,38 +170,36 @@ \section{Introduction} The presence of galaxies whose SEDs are not represented by the template library tends to lead to catastrophic outliers distributed like the horizontally oriented population of \Fig{fig:pedagogical_scatter}. For data-driven approaches, training sets that are incomplete in redshift coverage tend to result in catastrophic outliers like the vertically oriented population of \Fig{fig:pedagogical_scatter}. The approaches of using a training set versus a template library are related to one another by \citet{budavari_unified_2009}. -Sophisticated Bayesian techniques and machine learning methods have been employed to improve precision \citep{carliles_random_2010} and accuracy \citep{sadeh_annz2:_2016}, while other advances have focused on identifying and removing catastrophic outliers when using \pz s for inference \citep{gorecki_new_2014}. +Sophisticated Bayesian techniques and machine learning methods have been employed to improve precision \citep{carliles_random_2010} and accuracy \citep{sadeh_annz2:_2016}, while other advances have focused on identifying and removing catastrophic outliers when using \pz s for inference \citep{gorecki_new_2014}. % PDFs are a better way to improve photo-zs -\dwh{The probability density function (PDF) in redshift space for each galaxy, commonly written as $\pr{z}$, is an alternative to the MLE (with or without presumed Gaussian error bars) \citep{koo_photometric_1999}.} +The probability density function (PDF) in redshift space for each galaxy, commonly written as $\pr{z}$, is an alternative to the MLE (with or without presumed Gaussian error bars) \citep{koo_photometric_1999}. This option is favorable because it contains more potentially useful information about the uncertainty on each galaxy's redshift, incorporating our understanding of precision, accuracy, and systematic error. However, denoting \zpdf s as ``$\pr{z}$'' is an abuse of notation, as it does not adequately convey what information is being used to constrain the redshift $z$; \zpdf s are \textit{posterior} PDFs, conditioned on the photometric data and prior knowledge. In terms of \Fig{fig:pedagogical_scatter}, \zpdf s are horizontal cuts, probabilities of redshift conditioned on a specific value of data, i.e. posteriors $\pr{z \gvn \data}$, which constrain redshifts, whereas vertical cuts through this space are probabilities of data conditioned on a specific redshift, i.e. likelihoods $\pr{\data \gvn z}$, from which photometric data is actually drawn. % photo-z PDFs are established -\Pzpdf s have been produced by completed surveys \citep{hildebrandt_cfhtlens:_2012, sheldon_photometric_2012} and will be produced by ongoing and upcoming surveys \citep{abell_lsst_2009, carrasco_kind_exhausting_2014, bonnett_redshift_2016, masters_mapping_2015}. -\Pzpdf s are not without their own shortcomings, however, including the resources necessary to calculate and record them for large galaxy surveys \citep{carrasco_kind_sparse_2014, malz_approximating_2018} and the divergent results of each method used to derive them \citep[ and reviewed in Schmidt, Malz \& Soo, et al. (in prep)]{hildebrandt_phat:_2010, dahlen_critical_2013, sanchez_clustering_2013, bonnett_redshift_2016, tanaka_photometric_2018}. -The most concerning weakness of \pzpdf s, however, is their usage in the literature, which is at best inconsistent and at worst incorrect. +\Pzpdf s have been produced by completed surveys \citep{hildebrandt_cfhtlens:_2012, sheldon_photometric_2012} and will be produced by ongoing and upcoming surveys \citep{abell_lsst_2009, carrasco_kind_exhausting_2014, bonnett_redshift_2016, masters_mapping_2015}. +\Pzpdf s are not without their own shortcomings, however, including the resources necessary to calculate and record them for large galaxy surveys \citep{carrasco_kind_sparse_2014, malz_approximating_2018} and the divergent results of each method used to derive them \citep{hildebrandt_phat:_2010, dahlen_critical_2013, sanchez_clustering_2013, bonnett_redshift_2016, tanaka_photometric_2018}. +Though the matter is outside the scope of this paper, reviews of various methods have been presented in the literature \citep{sheldon_photometric_2012, ball_robust_2008, carrasco_kind_tpz:_2013, carrasco_kind_exhausting_2014, schmidt_evaluation_2020}. +The most concerning weakness of \pzpdf s, however, is their usage in the literature, which is at best inconsistent and at worst incorrect. % photo-z PDFs are most often reduced to point estimates Though their potential to improve estimates of physical parameters is tremendous, \pzpdf s have been applied only to a limited extent, most often by reduction to familiar point estimates. -If the true redshifts $\{z_{j}^{\dagger}\}$ of galaxies $j$ are known, then their \dwh{redshift PDFs are well-approximated by delta functions} $\{\delta(z, z_{j}^{\dagger})\}$ centered at the true redshift\dwh{\footnote{\dwh{Note that \sz s are not the same as known true redshifts; the PDFs of \sz s would be narrow and almost always unimodal, but they would not be delta functions due to observational errors.}}}, and the redshift distribution is effectively approximated by a histogram or other interpolation of the delta functions $\{\delta(z, z_{j}^{\dagger})\}$. +If the true redshifts $\{z_{j}^{\dagger}\}$ of galaxies $j$ are known, then their redshift PDFs are well-approximated by delta functions $\{\delta(z, z_{j}^{\dagger})\}$ centered at the true redshift\footnote{Note that \sz s are not the same as known true redshifts; the PDFs of \sz s would be narrow and almost always unimodal, but they would not be delta functions due to observational errors.}, and the redshift distribution is effectively approximated by a histogram or other interpolation of the delta functions $\{\delta(z, z_{j}^{\dagger})\}$. When \pzpdf s are available instead of true redshifts, the simplest approach reduces them to point estimates $\{\hat{z}_{i}\}$ of redshift by using $\delta(z, \hat{z}_{j})$ in place of $\delta(z, z_{j}^{\dagger})$. Though it is most common for $\hat{z}_{j}$ to be the maximum or \textit{mode} of the \pzpdf, there are other, more principled point estimate reduction procedures \citep{tanaka_photometric_2018}. % photo-z PDFs may also be used to define cuts Regardless of how it is done, any procedure that reduces \pzpdf s to point estimates discards valuable information about the uncertainty on redshift. -\Pzpdf s have also been used to form selection criteria of samples from galaxy surveys without propagation through the calculations of physical parameters \citep{van_breukelen_reliable_2009, viironen_high_2015}. +\Pzpdf s have also been used to form selection criteria of samples from galaxy surveys without propagation through the calculations of physical parameters \citep{van_breukelen_reliable_2009, viironen_high_2015}. Probability cuts on Bayesian quantities are not uncommon \citep{leung_bayesian_2017, dipompeo_quasar_2015}, but that procedure does not fully take advantage of all information contained in a probability distribution for parameter inference. -\aim{TODO: Provide equations/citations for the use of \Nz\ in cosmology, why it matters.} - The most prevalent application of \pzpdf s that preserves their information content is the estimation of the \textit{redshift distribution function \Nz}, or, interchangably, its normalized cousin the \textit{redshift density function \nz}. \nz\ is used to calculate the redshift calibration bias $b_{z}$ between the true and observed critical surface densities in galaxy-galaxy lensing \citep{mandelbaum_precision_2008} and the geometric lens efficiency $g_{k}(\chi)$ in tomographic weak lensing by large-scale structure \citep{benjamin_cfhtlens_2013}. -The redshift distribution is a key input to the traditional calculation of the power spectra of weak gravitational lensing and large-scale structure that are used to constrain the parameters of cosmological models \citep{bonnett_using_2015, masters_mapping_2015, viironen_high_2015, asorey_galaxy_2016, bonnett_redshift_2016, yang_calibrating_2018}. -\Nz\ may also be used to validate survey selection functions used in generation of realistic, multi-purpose mock catalogs \citep{norberg_2df_2002}. - -\aim{TODO: Say what precision is needed for \Nz\ for future weak lensing surveys, motivate how well we need to know \Nz.} +\Nz\ may be used to validate survey selection functions used in generation of realistic, multi-purpose mock catalogs \citep{norberg_2df_2002}. +As a key input to the traditional calculation of the power spectra of weak gravitational lensing and large-scale structure, the accuracy and precision to which \Nz\ is estimated can strongly impact our constraints on the parameters of cosmological models \citep{bonnett_using_2015, masters_mapping_2015, viironen_high_2015, asorey_galaxy_2016, bonnett_redshift_2016, yang_calibrating_2018}, so it is unsurprising that this last application dominates the canonical bias requirement of Table~\ref{tab:lsstsrd}. +%\aim{TODO: Say why \Nz\ matters for cosmology, what precision is needed for \Nz\ for future weak lensing surveys, motivate how well we need to know \Nz.} Even with \pz s adhering to the \lsst\ requirements of \Tab{tab:lsstsrd}, the degree to which constraints on the cosmological parameters can advance is limited by the accuracy and precision to which \nz\ is known \citep{abruzzo_impact_2019}. % Say what precision the mass function is needed (in, say cluster studies) for precision cosmology. @@ -205,29 +210,36 @@ \section{Introduction} \hat{n}(z) &= \frac{1}{J} \sum_{j = 0}^{J} \pr{z}_{j} \end{align} for a sample of $J$ galaxies $j$, or, equivalently, the redshift distribution function $\hat{N}(z) = J \hat{n}(z)$, by effectively averaging the \pzpdf s. -This summation procedure has been used extensively \citep{mandelbaum_precision_2008, benjamin_cfhtlens_2013, kelly_weighing_2014}\aim{\dots} +This summation procedure has been used extensively in cosmological analyses with photometric galaxy samples \citep{mandelbaum_precision_2008, benjamin_cfhtlens_2013, kelly_weighing_2014}. +%\aim{TODO: Continue adding equations/citations for the use of \Nz\ in cosmology.} % what this paper is about -Despite the growing prevalence of \pzpdf\ production, no implementation of inference using \pzpdf s has yet been presented with a mathematically consistent methodology. +Despite the growing prevalence of \pzpdf\ production, no implementation of inference using \pzpdf s has yet been presented with a mathematically consistent methodology. This paper challenges the logically invalid yet pervasive analysis procedure of stacking \pzpdf s by presenting and validating a hierarchical Bayesian technique for the use of \pzpdf s\ in the inference of \nz, yielding a method applicable to arbitrary one-point statistics relevant to cosmology, large-scale structure, and galaxy evolution; future work will extend this methodology to higher-order statistics. We aim to develop a clear methodology guiding the use of \pzpdf s in inference so they may be utilized effectively by the cosmology community. Though others have approached the problem before \citep{leistedt_hierarchical_2016, leistedt_hierarchical_2018}, the method presented here differs in that it makes use of any existing catalog of \pzpdf s, rather than requiring a simultaneous derivation of the \pzpdf s and the redshift distribution, making it preferable to ongoing surveys for which there may be inertia preventing a complete restructuring of the analysis pipeline. -In Section~\ref{sec:meth}, we present the \Chippr\ model and \chippr\ implementation for characterizing the full posterior probability landscape of \Nz\ using \pzpdf s. -In Section~\ref{sec:alldata}, we describe the experimental design for testing the fully probabilistic approach to mock and real datasets, the results of which are found in Section~\ref{sec:results}. -In Section~\ref{sec:results}, we stress-test the \Chippr\ model against stacking and its other competitors in the context of cosmology. +In Section~\ref{sec:meth}, we present the \Chippr\ model for characterizing the full posterior probability landscape of \Nz\ using \pzpdf s. +In Section~\ref{sec:application}, we present the \chippr\ implementation of the \Chippr\ model and the experimental set up by which we validate it, including the forward modeling of mock \pzpdf s. +In Section~\ref{sec:alldata}, we present a number of informative test cases and compare the results of \chippr\ with alternative approaches. +In Section~\ref{sec:results}, we stress-test the \Chippr\ model under nontraditional conditions. +% in the context of cosmology. +Finally, in Section~\ref{sec:con}, we make recommendations for future research involving \nz\ estimation. -\section{Method} +\section{Model} \label{sec:meth} -Consider a survey of $J$ galaxies $j$, each with photometric data $\data_{j}$; thus the entire survey over some solid angle produces the ensemble of photometric magnitudes (or colors) and their associated observational errors $\{\data_{j}\}$. -Each galaxy $j$ has a redshift $z_{j}$ that we would like to learn; redshift is a parameter in this case. -The distribution of the ensemble of redshifts $\{z_{j}\}$ may be described by the hyperparameters defining the redshift distribution function \nz\ that we would like to quantify. -This situation may be considered to be a probabilistic generative model, illustrated by the directed acyclic graph of \Fig{fig:pgm}. +Consider a survey of $J$ galaxies $j$, each with photometric data $\data_{j}$; +thus the entire survey over some solid angle produces the ensemble of photometric magnitudes (or colors) and their associated observational errors $\{\data_{j}\}$. +Each galaxy $j$ has a redshift parameter $z_{j}$ that we would like to learn. +The distribution of the ensemble of redshift parameters $\{z_{j}\}$ may be described by the hyperparameters defining the redshift distribution function \nz\ that we would like to quantify. +The redshift distribution function \nz\ is the number of galaxies per unit redshift, effectively defining the evolution in the number of galaxies convolved with the selection function of the sample \citep{menard_clustering-based_2013}. -The redshift distribution function \nz\ is the number of galaxies per unit redshift, effectively defining the evolution in the number of galaxies convolved with the selection function of the sample \citep{menard_clustering-based_2013}. -In the following sections, we present and compare methods for estimating \nz\ from \pzpdf s. -\Sect{sec:prob} contains the mathematical derivation of a probabilistic model for $n(z)$ dependent on photo-$z$ probability distribution functions, and \Sect{sec:sheldon} contrasts the probabilistic model with alternative methods. +In \Sect{sec:forward}, we establish a forward model encapsulating the causal relationship between \nz\ and photometry $\data$. +In \Sect{sec:prob}, we present the directed acyclic graph of this probabilistic generative model and interpret the corresponding mathematical expression, whose full derivation may be found in the Appendix. +In \Sect{sec:limitations}, we summarize the necessary assumptions of the model. + +%\aim{TODO: check for consistent notation of \pzip s vs. \pzpdf s.} \subsection{Forward Model} \label{sec:forward} @@ -239,7 +251,7 @@ \subsection{Forward Model} \int_{-\infty}^{\infty}\ n(z)\ dz\ \equiv\ \frac{1}{J}\ \int_{-\infty}^{\infty}\ \sum_{j=1}^{J}\ \delta(z_{j},\ z)\ dz = 1 \end{equation} of finding a galaxy $j$ in a catalog of $J$ galaxies having a redshift $z$. -We believe that galaxy redshifts are indeed drawn from \nz, making it a probability density over redshift; this fact can also be confirmed by dimensional analysis of \Eq{eqn:nz}, as suggested in \citet{hogg_data_2012}. +We believe that galaxy redshifts are indeed sampled, or drawn, from \nz, making it a probability density over redshift; this fact can also be confirmed by dimensional analysis of \Eq{eqn:nz}, as suggested in \citet{hogg_data_2012}. We may without loss of generality impose a parameterization \begin{equation} @@ -248,24 +260,25 @@ \subsection{Forward Model} \end{equation} in terms of some parameter vector $\ndphi$. At this point, the parameter vector is quite general and may represent coefficients in a high-order polynomial as a function of redshift, a set of means and variances defining Gaussians that sum to the desired distribution, a set of histogram heights that describe a binned version of the redshift distribution function, etc. -Upon doing so, we may rewrite \Eq{eqn:fz} as +Upon doing so, we may rewrite \Eq{eqn:fz} as \begin{equation} \label{eqn:pz} z_{j}\ \sim\ \pr{z \gvn \ndphi}\ \equiv\ f(z; \ndphi), \end{equation} -% \aim{explain "is drawn from"} a probability density over redshift conditioned on the parameters $\ndphi$ specifying \nz. Note that $z_{j}$ does not depend on the redshift $z_{j'}$ of some other galaxy $j' \neq j$, a statement of the causal independence of galaxy redshifts from one another. -In addition to believing \nz\ is a PDF from which redshifts are drawn, we also believe that there is some higher dimensional probability space $\pr{z, \data}$ of redshift and photometric data vectors $\data$, which may be any combination of fluxes, magnitudes, colors, and their observational errors. -In that sense \nz\ is equivalent to an integral +In addition to believing \nz\ is a PDF from which redshifts are drawn, we also believe that there is some higher dimensional probability space $\pr{z, \data}$ of redshift $z$ and photometric data vectors $\data$, which may be any combination of fluxes, magnitudes, colors, and their observational errors. +Under this framework, \nz\ is equivalent to an integral \begin{equation} \label{eqn:integral} n(z)\ =\ \integral{\pr{z, \data}}{\data} \end{equation} over the dimension of data in that joint probability space. -Note that galaxies may have different observational data despite sharing the same redshift, and that galaxies at different redshifts may have identical photometry; the space $\pr{z, \data}$ need not be one-to-one. -We assume a stronger version of statistical independence here, that draws $(z_{j}, \data_{j})$ are independent of draws $(z_{j'}, \data_{j'})$ in this space; the data and redshift of each galaxy are independent of those of other galaxies. +Note that galaxies may have different observational data despite sharing the same redshift, and that galaxies at different redshifts may have identical photometry; +the space $\pr{z, \data}$ need not be one-to-one. +We assume a stronger version of statistical independence here, that draws $(z_{j}, \data_{j})$ are independent of draws $(z_{j'}, \data_{j'})$ in this space; +the data and redshift of each galaxy are independent of those of other galaxies. However, this problem has additional causal structure that we can acknowledge. The photometry results from the redshifts, not the other way around. @@ -291,16 +304,15 @@ \subsection{Probabilistic Model} A DAG conveys the causal relationships between physical parameters and, like a Feynman diagram in the context of particle physics, is a shorthand for mathematical relationships between variables. The photometric data $\data_{j}$ of a galaxy is drawn from some function of its redshift $z_{j}$, independent of other galaxies' data and redshift. Both data and redshift are random variables, but data is the one that we observe and redshift is not directly observable. -In this problem, we don't care about further constraining the redshifts of individual galaxies, only the redshift distribution, so we consider redshift to be a \textit{latent variable}. +In this problem, we don't care about further constraining the redshifts of individual galaxies, only the redshift distribution \nz, so we consider redshift to be a \textit{latent variable}. Because the parameters $\ndphi$ that we seek are causally separated from the data by the latent variable of redshift, we call them \textit{hyperparameters}. -\que{Am I adequately explaining the distinction between hyperparameters and parameters?} \begin{figure} \begin{center} \includegraphics[height=0.25\textheight]{figures/chippr/pgm.png} \caption{The directed acyclic graph of the CHIPPR model, where circles indicate random variables and arrows indicate causal relationships. - The redshift distribution \nz\ parameterized by hyperparameters $\ndphi$ exists independent of the survey of $J$ galaxies, indicated as a box. - The redshifts $\{z_{j}\}$ of all galaxies in the survey are latent variables independently drawn from the redshift distribution, which is a function of $\ndphi$. + The redshift distribution \nz\ parameterized by hyperparameters $\ndphi$ exists independent of the survey of $J$ galaxies, indicated as a box. + The redshifts $\{z_{j}\}$ of all galaxies in the survey are latent variables independently drawn from the redshift distribution, which is a function of $\ndphi$. The photometric data $\data_{j}$ for each galaxy is drawn from a function of its redshift $z_{j}$ and observed, indicated by a shaded circle. } \label{fig:pgm} @@ -309,58 +321,81 @@ \subsection{Probabilistic Model} The problem facing cosmologists is to determine the true value of $\ndphi$ from observing the photometry $\{\data_{j}\}$ of a large sample of $J$ galaxies $j$. To self-consistently propagate the uncertainty in the inference of redshift, however, it is more appropriate to estimate the posterior $\pr{\ndphi \gvn \{\data_{j}\}}$ over all possible values of $\ndphi$ conditioned on all the observed data $\{\data_{j}\}$ available in a generic catalog. -In order to use the DAG of \Fig{fig:pgm} to derive an expression for $\pr{\ndphi \gvn \{\data_{j}\}}$ in terms of \pzpdf s, we must introduce two more concepts, confusingly named the \textit{implicit prior} and the \textit{prior probability density (prior PDF)}, elaborated upon below. +In order to use the DAG of \Fig{fig:pgm} to derive an expression for $\pr{\ndphi \gvn \{\data_{j}\}}$ in terms of \pzpdf s, we must introduce two more concepts, confusingly named the \textit{implicit prior} and the \textit{prior probability density} (\textit{prior PDF}), elaborated upon below. + +When we constrain the redshift of a galaxy using its observed photometric data $\data_{j}$, we are effectively estimating a posterior $\pr{z \gvn \data_{j}}$, the probability of an unknown quantity conditioned on the quantity we have in hand, i.e the photometric data. +This posterior is effectively a marginalization with respect to redshift at a given value of $\data = \data_{j}$ of the \textit{empirical frequency distribution} $\pr{z, \data \gvn \ndphi^{\dagger}}$, the joint probability density corresponding to the true redshift distribution parameterized by $\ndphi^{\dagger}$, which exists in nature but need not be known. +%\que{Propagate new $\pr{z, \data \gvn \ndphi^{\dagger}}$ notation through appendix?} -\dwh{When we constrain the redshift of a galaxy using its observed photometric data $\data_{j}$, we are effectively estimating a posterior $\pr{z \gvn \data_{j}}$, the probability of an unknown quantity conditioned on the quantity we have in hand, i.e the data. -However, to do this, we must have a model for the general relationship between redshifts and photometry, whether empirical, as is the case for machine learning \pzpdf\ methods, or analytic, as is the case for template-based \pzpdf\ methods. -Such a relationship $\pr{\data, z}$ is defined in the multidimensional space of probability density over redshift and photometry. +As the hyperparameters $\ndphi^{\dagger}$ of the true redshift distribution are in general unknown, the investigator seeking to estimate a posterior $\pr{z \gvn \data_{j}}$ must have a model $\phi^{*}$ for the general relationship between redshifts and photometry, whether empirical, as is the case for machine learning \pzpdf\ methods, or analytic, as is the case for template-based \pzpdf\ methods. If we were to marginalize over the photometry in $\pr{\data, z}$, we would obtain a one-dimensional PDF $\pr{z \gvn \ndphi^{*}}$ over redshift, which can by definition be parameterized by the same functional form as \nz, for some $\ndphi^{*}$ specific to the estimation procedure that may or may not bear any relation to the hyperparameters $\ndphi^{\dagger}$ of the true \nz. -Rather, $\ndphi^{*}$ is a consequence of generative model for how photometry results from redshift, including the influence of intrinsic galaxy spectra and instrumental effects. -We call $\pr{z \gvn \ndphi^{*}}$ the \textit{implicit prior}, as it is rarely explicitly known nor chosen by the researcher}\footnote{For template-based methods, the implicit prior is often an explicitly known input to the algorithm, engineered as an initial guess for the true $\ndphi$, with an aim for a realistic choice guided by an earlier spectroscopic survey. +Rather, $\ndphi^{*}$ is a consequence of the generative model for how photometry results from redshift, including the influence of intrinsic galaxy spectra and instrumental effects. + +We call $\pr{z \gvn \ndphi^{*}}$ the \textit{implicit prior}, as it is rarely explicitly known nor chosen by the researcher\footnote{For template-based methods, the implicit prior is often an explicitly known input to the algorithm, engineered as an initial guess for the true $\ndphi$, with an aim for a realistic choice guided by an earlier spectroscopic survey. (See \citet{benitez_bayesian_2000} for more detail.) It may thus be more appropriate to call it an \textit{interim prior}, but we will use the former term throughout this paper for generality.} Because the implicit prior is unavoidable and almost inherently not uninformative, the \pzpdf s reported by any method must be \textit{implicit posteriors} ${\pr{z \gvn \data, \ndphi^{*}}}$ weighted by the implicit prior. -%Posteriors differ from likelihoods by way of a prior distribution, so we cannot simply assume that the available data products are \pz\ posteriors $\pr{z \gvn \data_{j}}$. -%Rather, we have a catalog of implicit-prior weighted \pz\ posteriors $\pr{z \gvn \data_{j}, \ndphi^{*}}$. -%There must have been some interim prior probability distribution $p(z|\vec{\theta}^{*})$ defined in terms of the interim prior parameter values (hereafter the interim prior) $\vec{\theta}^{*}$ explicitly chosen or implicitly made to perform the calculation of the probabilistic photo-$z$s. -%If it is implicit, it may not be representable in the parametrization we have chosen, and furthermore it may not be known at all; a method that produces interim photo-$z$ posteriors of this kind is not suitable for inference. -%However, so long as the implicit prior is known, hierarchical inference is possible. +%Posteriors differ from likelihoods by way of a prior distribution, so we cannot simply assume that the available data products are \pz\ posteriors $\pr{z \gvn \data_{j}}$. +%Rather, we have a catalog of implicit-prior weighted \pz\ posteriors $\pr{z \gvn \data_{j}, \ndphi^{*}}$. +%There must have been some interim prior probability distribution $p(z|\vec{\theta}^{*})$ defined in terms of the interim prior parameter values (hereafter the interim prior) $\vec{\theta}^{*}$ explicitly chosen or implicitly made to perform the calculation of the probabilistic photo-$z$s. +%If it is implicit, it may not be representable in the parametrization we have chosen, and furthermore it may not be known at all; a method that produces interim photo-$z$ posteriors of this kind is not suitable for inference. +%However, so long as the implicit prior is known, hierarchical inference is possible. The prior probability density $\pr{\ndphi}$ is a more familiar concept in astronomy; to progress, we will have to choose a prior probability density over all possible values of the hyperparameters $\ndphi$. This prior need not be excessively proscriptive; for example, it may be chosen to enforce smoothness at physically motivated scales in redshift without imposing any particular region as over- or under-dense. -\dwh{With inputs of the \pzip\ catalog $\{\pr{z \gvn \data, \ndphi^{*}}\}$, the implicit prior $\pr{z \gvn \ndphi^{*}}$, and the prior PDF $\pr{\ndphi}$, we thus aim to obtain the posterior probability $\pr{\ndphi \gvn \{\data_{j}\}}$ of the redshift density function given all the photometric data. -By performing the derivation of Appendix~\ref{app:math}, we arrive at the desired expression +With inputs of the \pzip\ catalog $\{\pr{z \gvn \data, \ndphi^{*}}\}$, the implicit prior $\pr{z \gvn \ndphi^{*}}$, and the prior PDF $\pr{\ndphi}$, we thus aim to obtain the posterior probability $\pr{\ndphi \gvn \{\data_{j}\}}$ of the redshift density function given all the photometric data. +By performing the derivation of the Appendix, we arrive at the desired expression \begin{equation} \label{eqn:fullpost} \pr{\ndphi \gvn \{\data_{j}\}} \propto \pr{\ndphi} \integral{\prod_{j=1}^{J} \frac{\pr{z \gvn \data_{j}, \ndphi^{*}} \pr{z \gvn \ndphi}}{\pr{z \gvn \ndphi^{*}}}}{z}, \end{equation} -which is the very heart of \Chippr. -This in effect replaces the implicit prior with the sampled model hyperparameters, thereby converting the \pzip s into likelihoods in order to obtain unbiased posteriors.} +which is the very heart of \Chippr, also given as \Eq{eqn:final}. +This in effect replaces the implicit prior with the sampled model hyperparameters, thereby converting the \pzip s into likelihoods in order to obtain unbiased posteriors. -\subsubsection{Model Limitations} +\subsection{Model Limitations} \label{sec:limitations} Finally, we explicitly review the assumptions made by this approach, which are as follows: \begin{enumerate} \item Photometric measurements of galaxies are statistically independent Poisson draws from the set of all galaxies such that \Eq{eqn:indiedat} and \Eq{eqn:indie} hold. - \item We take the reported \pzip s to be accurate, free of model misspecification; draws thereof must not be inconsistent with the distribution of photometry and redshifts. + \item We take the reported \pzip s to be accurate, free of model misspecification; + draws thereof must not be inconsistent with the distribution of photometry and redshifts. Furthermore, we must be given the implicit prior $\ndphi^{*}$ used to produce the \pzip s. \item We must assume a hyperprior distribution $\pr{\ndphi}$ constraining the underlying probability distribution of the hyperparameters, which is informed by our prior beliefs about the true redshift distribution function. \end{enumerate} -These assumptions have known limitations. -First, the photometric data are not a set of independent measurements; the data are correlated not only by the conditions of the experiment under which they were observed (instrument and observing conditions) but also by redshift covariances resulting from physical processes governing underlying galaxy spectra and their relation to the redshift distribution function. -Second, the reported \pzip s may not be trustworthy; there is not yet agreement on the best technique to obtain \pzpdf s, and the implicit prior may not be appropriate or even known to us as consumers of \pzip s. +These assumptions have known limitations. +First, the photometric data are not a set of independent measurements; +the data are correlated not only by the conditions of the experiment under which they were observed (instrument and observing conditions) but also by redshift covariances resulting from physical processes governing underlying galaxy spectra and their relation to the redshift distribution function. +Second, the reported \pzip s may not be trustworthy; +there is not yet agreement on the best technique to obtain \pzpdf s, and the implicit prior may not be appropriate or even known to us as consumers of \pzip s. Third, the hyperprior may be quite arbitrary and poorly motivated if the underlying physics is complex, and it can only be appropriate if our prior beliefs about \nz\ are accurate. +Furthermore, in Section~\ref{sec:prob}, we have made an assumption of \textit{support}, meaning the model $\pr{z, \data \gvn \ndphi}$ has mutual coverage with the parameter values that real galaxies can take. +In other words, any probability distribution over the $(z, \data)$ space must be nonzero where real galaxies can exist. +Additionally, the hyperprior $\pr{\ndphi}$ must be nonzero at the hyperparameters $\ndphi^{\dagger}$ of the true redshift density function \nz. +This assumption cannot be violated under the experimental design of Section~\ref{sec:forward}, but it is not generically guaranteed when performing inference on real data; +thus the chosen $\pr{z, \data \gvn \ndphi^{*}}$ and $\pr{\ndphi}$ must be sufficiently general as to not rule out plausible areas of parameter space. + +\section{Methods \& Data} +\label{sec:application} + +Here we describe the method by which we demonstrate the \Chippr\ model. +In \Sect{sec:exp}, we outline the implementation of the \chippr\ code. +%In \Sect{sec:sheldon}, we introduce the alternative \nz\ estimators against which \Chippr\ is compared. +%In \Sect{sec:diag}, we present the quantitative metrics by which the \nz\ estimators are compared. +In \Sect{sec:mock}, we outline the procedure for emulating mock \pzip s. + \subsection{Implementation} \label{sec:exp} We implement the \Chippr\ model in code in order to perform tests of its validity and to compare its performance to that of traditional alternatives. In \Sect{sec:mcmc}, we describe the publicly available \chippr\ library. -In \Sect{app:acorr}, we outline how \chippr\ can be used to sample the full log-posterior distribution $\ln[\pr{\ndphi \gvn \{\data_{j}\}}]$. +In \Sect{sec:sheldon}, we introduce the alternative approaches evaluated for comparison with \Chippr. +In \Sect{sec:diag}, we describe the diagnostic criteria by which we assess estimators of \nz. +%In \Sect{app:acorr}, we outline how \chippr\ can be used to sample the full log-posterior distribution $\ln[\pr{\ndphi \gvn \{\data_{j}\}}]$. \subsubsection{Code} \label{sec:mcmc} @@ -378,106 +413,109 @@ \subsubsection{Code} \sum_{k=1}^{K} \exp[\phi^{k}] \delta z^{k} = 1 \end{equation} with $K$ bins of width $\delta z^{1}, \dots, \delta z^{K}$ defined by endpoints $z^{0}, \dots, z^{K}$. -%\aim{Maybe include an equation for a general piecewise constant function here (which is currently in \Chap{qp})?} Thus each $\pr{z \gvn \data_{j}} = f(z; \ndphi_{j})$ has parameters $\ndphi_{j}$ that are defined in the same basis as those of \nz. To infer the full log-posterior distribution $\ln[\pr{\ndphi \gvn \{\data_{j}\}}]$, one must provide a plaintext file with $K+1$ redshift bin endpoints $\{z_{k}\}$, the parameters $\ndphi^{*}$ of the implicit log-prior, and the parameters $\{\ndphi_{j}\}$ of the log-posteriors $\ln[\pr{z \gvn \data_{j}, \ndphi^{*})}$. -The \emcee \citep{foreman-mackey_emcee_2013} implementation of ensemble sampling is used to sample the full log-posterior of \Eq{eqn:final}. +The \emcee \citep{foreman-mackey_emcee_2013} implementation of ensemble sampling is used to sample the full log-posterior of \Eq{eqn:final}. \chippr\ accepts a configuration file of user-specified parameters, among them the number $W$ of walkers. At each iteration $i$ and for each walker, a proposal distribution $\hat{\ndphi}_{i}$ is drawn from the log-prior distribution and evaluated for acceptance to or rejection from the full log-posterior distribution. +%Two threshold conditions are defined, one designating all previous samples to be ignored as as products of a burn-in phase and another indicating when a sufficient number of post-burn samples have been accepted. +%In this case, the first threshold (described in \Sect{app:acorr}) is defined in terms of sub-runs of $10^{3}$ accepted samples, and the second is defined as an accumulation of $10^{4}$ samples. +%Though previous versions used \texttt{HDF5} for the primary I/O format due to its efficiency for large quantities of data, it was abandoned in favor of \texttt{pickle} in the working release due to the instability of the \python\ implementation of the format on high-performance computing systems. -Two threshold conditions are defined, one designating all previous samples to be ignored as as products of a burn-in phase and another indicating when a sufficient number of post-burn samples have been accepted. -In this case, the first threshold (described in \Sect{app:acorr}) is defined in terms of sub-runs of $10^{3}$ accepted samples, and the second is defined as an accumulation of $10^{4}$ samples. +The resulting output +% is a set of ordered \texttt{pickle} files % enumerated by $\rho$ each containing the state information of $10^{3}$ accepted samples, with the last ten files including only samples taken after the completion of the burn-in phase +% after each sub-run. +%The state information +includes $\frac{I_{0}}{s}$ accepted samples $\ndphi_{i}$ for a pre-specified chain thinning factor $s$ and their full posterior probabilities $\pr{\ndphi_{i} \gvn \{\data_{j}\}}$, as well as the autocorrelation times and acceptance fractions calculated for each element of $\ndphi$, divided into separate files before and after the completion of the burn-in phase, as defined by the Gelman-Rubin statistic \citep{gelman_inference_1992}. -%Though previous versions used \texttt{HDF5} for the primary I/O format due to its efficiency for large quantities of data, it was abandoned in favor of \texttt{pickle} in the working release due to the instability of the \python\ implementation of the format on high-performance computing systems. -The resulting output is a set of $I$ ordered \texttt{pickle} files enumerated by $\rho$ containing the state information after each sub-run. -The state information includes $\frac{I_{0}}{s}$ actual samples $\ndphi_{i}$ for a pre-specified chain thinning factor $s$ and their full posterior probabilities $\pr{\ndphi_{i} \gvn \{\data_{j}\}}$ as well as the autocorrelation times and acceptance fractions calculated for each element of $\ndphi$ over the entire sub-run. - -\subsection{Comparison with Alternative Approaches} +\subsubsection{Alternative approaches for comparison} \label{sec:sheldon} -In this study, we compare the results of \Eq{eqn:fullpost} to those of the two most common approaches to estimating \nz\ from a catalog of \pzpdf s: the distribution $n(z_{\mathrm{max}})$ of the redshifts at maximum posterior probability +In this study, we compare the results of \Eq{eqn:fullpost} to those of the two most common approaches to estimating \nz\ from a catalog of \pzip s: +the distribution $n(z_{\mathrm{max}})$ of the redshifts at maximum posterior probability \begin{equation} \label{eqn:mmap} f^{MMAP}(z; \hat{\ndphi}) = \sum_{j=1}^{J}\ \delta(z, \mathrm{mode}[\pr{z \gvn \data_{j}, \ndphi^{*}}]) \end{equation} -(i.e. the distribution of modes of the \pzpdf s) and the stacked estimator of \Eq{eqn:stacked}, which can be rewritten as +(i.e. the distribution of modes of the \pzip s) and the stacked estimator of \Eq{eqn:stacked}, which can be rewritten as \begin{equation} \label{eqn:stacked} f^{stack}(z; \hat{\ndphi}) = \sum_{j=1}^{J}\ \pr{z \gvn \data_{j}, \ndphi^{*}} \end{equation} -in terms of the implicit \pz\ posteriors we have. +in terms of the \pzip s we have. These two approaches have been compared to one another by \citet{hildebrandt_cfhtlens:_2012}, \citet{benjamin_cfhtlens_2013}, and \citet{asorey_galaxy_2016} in the past but not to \Chippr. -Point estimation converts the implicit photo-$z$ posteriors $\pr{z \gvn \data_{j}, \ndphi^{*}}$ into delta functions with all probability at a single estimated redshift. +Point estimation converts the implicit \pz\ posteriors $\pr{z \gvn \data_{j}, \ndphi^{*}}$ into delta functions with all probability at a single estimated redshift. Some variants of point estimation choose this single redshift to be that of maximum a posteriori probability $\mathrm{mode}[\pr{z \gvn \data_{j}, \ndphi^{*}}]$ or the expected value of redshift $\langle z \rangle = \integral{z \pr{z \gvn \data_{j}, \ndphi^{*}}}{z}$. \citet{tanaka_photometric_2018} directs attention to deriving an optimal point estimate reduction of a \pzpdf, but since the purpose of this paper is to compare against the most established alternative estimators of \nz, its use will be postponed until a future study. -Stacking these modified \pzpdf s leads to the marginalized maximum a posteriori (MMAP) estimator and the marginalized expected value (MExp) estimator, though only the former is included in this study since the latter has fallen out of favor in recent years\footnote{And for good reason! Consider a bimodal \pzpdf; its expected value may very well fall in a region of very low probability, yielding a less probable point estimate than the point at which either peak achieves its maximum.}. +Stacking these modified \pzip s leads to the marginalized maximum a posteriori (MMAP) estimator and the marginalized expected value (MExp) estimator, though only the former is included in this study since the latter has fallen out of favor in recent years\footnote{And for good reason! Consider a bimodal \pzpdf; its expected value may very well fall in a region of very low probability, yielding a less probable point estimate than the point at which either peak achieves its maximum.}. -It is worth discussing the relationship between point estimation and stacking. -When the point estimator of redshift is equal to the true redshift, stacking delta function \pzpdf s will indeed lead to an accurate recovery of the true redshift distribution function. -However, stacking is in general applied indiscriminately to broader \pzpdf s and imperfect point estimators of redshift. +It is worth discussing the relationship between point estimation and stacking. +When the point estimator of redshift is equal to the true redshift, stacking delta function \pzpdf s will indeed lead to an accurate recovery of the true redshift distribution function. +However, stacking is in general applied indiscriminately to broader \pzpdf s and imperfect point estimators of redshift. It is for these reasons that alternatives are considered here. -A final estimator of the hyperparameters is the maximum marginalized likelihood estimator, the value of $\ndphi$ maximizing the log posterior given by \Eq{eqn:final} using any optimization code. -To compare with sampling, the MMLE also depends on the choice of the hyperprior distribution, and it does not produce a full posterior probability distribution over the parameters of interest, only point estimators. -It must be noted that computation of the maximum marginalized likelihood estimator may be unstable depending on the strengths and weaknesses of the optimizer. -In general, derivatives will not be available for the full posterior distribution, restricting optimization methods used. +A final estimator of the hyperparameters is the maximum marginalized likelihood estimator (MMLE), the value of $\ndphi$ maximizing the log posterior given by \Eq{eqn:final} using any optimization code. +The MMLE can be obtained in substantially less time than enough samples to characterize the full log-posterior distribution of \nz. +However, the MMLE yields only a point estimate of \nz\ rather than characterizing the full log-posterior on $\ndphi$, and it does not escape the dependence on the choice of hyperprior distribution. +Furthermore, derivatives will not in general be available for the full posterior distribution, restricting optimization methods used, and, as is true for any optimization code, there is a risk of numerical instability. %\begin{equation} %\label{eq:mmle} -%\ln[p(\{\vec{d}_{j}\}|\vec{\theta})] \propto -\int\ f_{\vec{\theta}}(z)\ -%dz+\sum_{j=1}^{J}\ln\left[\int\ +%\ln[p(\{\vec{d}_{j}\}|\vec{\theta})] \propto -\int\ f_{\vec{\theta}}(z)\ +%dz+\sum_{j=1}^{J}\ln\left[\int\ %\exp\left[\ln[p(z_{j}|\vec{d}_{j},\vec{\theta}^{*})]+\ln[f_{\vec{\theta}}(z)]-\l %n[f_{\vec{\theta}^{*}}(z)]\right]\ dz\right], %\end{equation} %accessible with any optimization code. -In \Sect{sec:diag} we outline the measures used to evaluate the performance of the method. - \subsubsection{Performance metrics} \label{sec:diag} -The results of the computation described in \Sect{sec:mcmc} are evaluated for accuracy on the basis of some quantitative measures. -Beyond visual inspection of samples, we calculate summary statistics to quantitatively compare different estimators' precision and accuracy. -Since MCMC samples of hyperparameters are Gaussian distributions, we can quantify the breadth of the distribution for each hyperparameter using the standard deviation regardless of whether the true values are known. +The results of the computation described in \Sect{sec:exp} are evaluated for accuracy on the basis of some quantitative measures. +Beyond visual inspection of samples, we calculate summary statistics to quantitatively compare different estimators' precision and accuracy. +Since MCMC samples of hyperparameters are Gaussian distributions, we can quantify the breadth of the distribution for each hyperparameter using the standard deviation regardless of whether the true values are known. -In simulated cases where the true parameter values are known, we calculate the Kullback-Leibler divergence (KLD), given by +In simulated cases where the true parameter values are known, we calculate the Kullback-Leibler divergence (KLD), given by \begin{equation} \label{eqn:kl} -KL_{',\ddagger} = \integral{\pr{z \gvn \ndphi'} \ln \left[ \frac{\pr{z \gvn \ndphi'}}{\pr{z \gvn \ndphi^{\dagger}}} \right]}{z} , +KL_{\ndphi,\ndphi^{\ddagger}} = \integral{\pr{z \gvn \ndphi} \ln \left[ \frac{\pr{z \gvn \ndphi}}{\pr{z \gvn \ndphi^{\dagger}}} \right]}{z} , \end{equation} -which measures a distance from parameter values $\ndphi'$ to true parameter values $\ndphi^{\dagger}$, which is invariant under changes of variables. -We note that $KL_{',\dagger} \neq KL_{\dagger,'}$ and is only interpretable when there is a notion that $\ndphi^{\dagger}$ is closer to the truth than $\ndphi'$. -The KLD is explored in detail in the Appendix to \citet{malz_approximating_2018}. -In simulated tests, $\ndphi^{\dagger}$ is the true value and $\ndphi'$ is that produced by one of the methods in question. +which measures a distance from parameter values $\ndphi$ to true parameter values $\ndphi^{\dagger}$. +The KLD is a measure of the information loss, in units of nats, due to using $\ndphi$ to approximate the true $\ndphi^{\dagger}$ when it is known. +A detailed exploration of the KLD may be found in the Appendix to \citet{malz_approximating_2018}. +%We note that $KL_{\ndphi,\ndphi^{\ddagger}} \neq KL_{\ndphi^{\ddagger},\ndphi}$ and is only interpretable when there is a notion that $\ndphi^{\dagger}$ is closer to the truth than $\ndphi$. +%In simulated tests, $\ndphi^{\dagger}$ is the true value and $\ndphi'$ is that produced by one of the methods in question. -\section{Validation} -\label{sec:alldata} - -We compare the results of \Chippr\ to those of stacking and the histogram of \pzpdf\ maxima (modes) on mock data in the form of catalogs of emulated \pzpdf s generated via the forward model discussed in \Sect{sec:forward}. -\dwh{\Fig{fig:flowchart} illustrates the implementation of the forward model, defined by the much simpler \Fig{fig:pgm}, used for validating the method presented here. -The irony of a simple model and complex validation procedure is not lost on the authors.} +%\aim{TODO: Also include the mean redshift/$\Delta_{z}$ for each estimator of \nz, cite the DES/KiDS/HSC papers that motivate this.} -\dwh{\Fig{fig:flowchart} outlines the four phases of the generative model, which uses a total of three inputs. -The experimental design requires our choice of true values $\phi^{\dagger}$ of the hyperparameters governing \nz, a \pz\ model $\pr{z, \data}$ defining the space of redshift and photometry, and prior values $\phi^{*}$ of the hyperparameters of \nz. -In the first phase, we sample $J = 10^{4}$ redshifts $z_{j}^{\dagger} \sim \pr{z \gvn \phi^{\dagger}}$. -In the second phase, we evaluate the \pz\ model at those redshifts, yielding a set of $J$ likelihoods $\pr{\data \gvn z_{j}^{\dagger}}$, from which we then sample data $\data_{j}^{\dagger} \sim \pr{\data \gvn z_{j}^{\dagger}}$ for each galaxy. -In the third phase, we evaluate the \pz\ model at that data to obtain $J$ posteriors $\pr{z \gvn \data_{j}^{\dagger}}$. -In the fourth phase, we convolve the posteriors with the chosen prior $\pr{z \gvn \phi^{*}}$, yielding implicit posteriors $\pr{z \gvn \data_{j}^{\dagger}, \phi^{*}}$.} +\subsection{Validation on mock data} +\label{sec:mock} \begin{figure*} \begin{center} - \includegraphics[width=0.9\textwidth]{figures/chippr/flowchart.pdf} + \includegraphics[width=0.7\textwidth]{figures/chippr/flowchart.pdf} \caption{ - \que{How to make this more attractive?} + % \que{Change ``true joint model'' to true forward model, condition on $\phi^{\dagger}$, add footnote explaining distinction between dagger, star, prime, etc., check that derivation matches notation} A flow chart illustrating the forward model used to generate mock data in the validation of \Chippr, as described in \Sect{sec:forward}. Ovals indicate a quantity that must be chosen in order to generate the data, rectangles indicate an operation we perform, and rounded rectangles indicate a quantity created by the forward model. - Arrows indicate the inputs and outputs of each operation performed to simulate mock \pzpdf\ catalogs. - } + Arrows indicate the inputs and outputs of each operation performed to simulate mock \pzip\ catalogs. + } \label{fig:flowchart} \end{center} \end{figure*} +We compare the results of \Chippr\ to those of stacking and the histogram of \pzip\ maxima (modes) on mock data in the form of catalogs of emulated \pzip s generated via the forward model discussed in \Sect{sec:forward}. +\Fig{fig:flowchart} illustrates the implementation of the forward model, defined by the much simpler \Fig{fig:pgm}, used for validating the method presented here. +The irony of a simple model and complex validation procedure is not lost on the authors. + +\Fig{fig:flowchart} outlines the four phases of the generative model, which uses a total of three inputs. +The experimental design requires our choice of true values $\phi^{\dagger}$ of the hyperparameters governing \nz, a \pz\ model $\pr{z, \data}$ defining the space of redshift and photometry, and prior values $\phi^{*}$ of the hyperparameters of \nz. +In the first phase, we sample $J = 10^{4}$ redshifts $z_{j}^{\dagger} \sim \pr{z \gvn \phi^{\dagger}}$. +In the second phase, we evaluate the \pz\ model at those redshifts, yielding a set of $J$ likelihoods $\pr{\data \gvn z_{j}^{\dagger}}$, from which we then sample data $\data_{j}^{\dagger} \sim \pr{\data \gvn z_{j}^{\dagger}}$ for each galaxy. +In the third phase, we evaluate the \pz\ model at that data to obtain $J$ posteriors $\pr{z \gvn \data_{j}^{\dagger}}$. +In the fourth phase, we convolve the posteriors with the chosen prior $\pr{z \gvn \phi^{*}}$, yielding implicit posteriors $\pr{z \gvn \data_{j}^{\dagger}, \phi^{*}}$. + The true redshift distribution used in these tests is a particular instance of the gamma function \begin{equation} \label{eqn:gamma} @@ -486,69 +524,64 @@ \section{Validation} with $c_{z} = 0.3$, because it has been used in forecasting studies for \des\ and \lsst. %\aim{I learned this from talking to people and don't know of a published source that talks about the nitty gritty of the internal validation tests performed before there was data.} -The mock data emulates the three sources of error of highest concern to the \pz\ community that are explored in detail later in this section: intrinsic scatter (\Sect{sec:scatter}), catastrophic outliers (\Sect{sec:outliers}), and systematic bias (\Sect{sec:bias}). -Tests including all three effects at the tolerance levels of \lsst\ (see Table~\ref{tab:lsstsrd}) -are presented in \Sect{sec:results}. -\Fig{fig:mega_scatter} illustrates these three effects individually at twice the tolerance of \lsst\ for demonstrative purposes, hearkening back to Figure~\ref{fig:pedagogical_scatter}. -We also test nontrivial implicit priors in \Sect{sec:interim}, which ought to be a priority for the \pz\ community. +The mock data emulates the three sources of error of highest concern to the \pz\ community that are explored in detail later in this section: intrinsic scatter (\Sect{sec:scatter}), catastrophic outliers (\Sect{sec:outliers}), and canonical bias (\Sect{sec:bias}). +\Fig{fig:mega_scatter} illustrates these three effects simultaneously at the tolerance of \lsst\ for demonstrative purposes, harking back to Figure~\ref{fig:pedagogical_scatter}. -\begin{figure*} +\begin{figure}%[H] \begin{center} - \includegraphics[width=0.32\textwidth]{figures/chippr/thesis_hivarsig-mega_scatter.png} - \includegraphics[width=0.32\textwidth]{figures/chippr/single_uout-mega_scatter.png} - \includegraphics[width=0.32\textwidth]{figures/chippr/thesis_neghivarbias-mega_scatter.png} - \caption{The joint probability space of true and estimated redshift for the three concerning \pz\ systematics at twice the level of the \lsst\ requirements: intrinsic scatter (left), uniform outliers (center), and bias (right). - The main panel of each shows samples (black points) in the space of mock data and redshift, akin to the standard scatterplots of true and estimated redshift, the $z_{\mathrm{true}} = z_{\mathrm{phot}}$ diagonal (thin red line), and posterior probabilities evaluated at the given estimated redshift (colored step functions). - The insets show marginal histograms (gray) in each dimension that can be compared with the true \nz\ (blue curve) used to make the figure to see the effect of the isolated systematic, as well as the implicit prior (red line). - \aim{TODO: Omit this figure; redundant with right panel of \Fig{fig:pzs-scatter} and top panels of \Fig{fig:uniform-outliers} and \Fig{fig:bias}.} -%% Enlarge axis labels and label panels.} -%% Add watermark of ``mock data'' in UL corner, same for ``results of inference'' on other kind of plot.} + \includegraphics[width=0.45\textwidth]{figures/chippr/single_lsst-mega_scatter.png} + \caption{ + The joint probability space of true and estimated redshift for the three concerning \pz\ systematics at the level of the \lsst\ requirements: + intrinsic scatter, uniformly distributed catastrophic outliers, and bias. + The main panel shows samples (black points) in the space of mock data and redshift, akin to the standard scatterplots of true and estimated redshift, the $z_{\mathrm{spec}} = z_{\mathrm{phot}}$ diagonal (gray line), and posterior probabilities evaluated at the given estimated redshift (colored step functions). + The insets show marginal histograms (light gray) in each dimension, that can be compared with the true \nz\ used to make the figure (black) to see the effect of these systematics, as well as the implicit prior (dark gray). +% \aim{TODO: Include slice further up so we can see outliers. +% Label panel.} } \label{fig:mega_scatter} \end{center} -\end{figure*} +\end{figure} The hyperprior distribution chosen for these tests is a multivariate normal distribution with mean $\vec{\mu}$ equal to the implicit prior $\ndphi^{*}$ and covariance \begin{equation} \label{eqn:priorcov} \Sigma_{k,k'} = q\ \exp[-\frac{e}{2}\ (\bar{z}_{k}-\bar{z}_{k'})^{2}]\ +\ t\delta(k,k') \end{equation} -inspired by one used in Gaussian processes, where $k$ and $k'$ are indices ranging from $1$ to $K$ and $q=1.0$, $e=100.0$, and $t=q\cdot10^{-5}$ are constants chosen to permit draws from this prior distribution to produce shapes similar to that of a true $\tilde{\ndphi}$. +inspired by one used in Gaussian processes, where $k$ and $k'$ are indices ranging from $1$ to $K$ and $q=1.0$, $e=100.0$, and $t=q\cdot10^{-5}$ are constants chosen to permit draws from this prior distribution to produce shapes similar to that of a true $\tilde{\ndphi}$. We adapt the full log-posterior of \Eq{eqn:final} to the chosen binning of redshift space. %An example of such samples from the prior are shown in \Fig{fig:prior}. -% -%\begin{figure} -%% \includegraphics[width=0.5\textwidth]{figures/chippr/null_priorsamps.pdf} -% \caption{\aim{I need to remake this one because it uses the wrong notation and I stopped making it automatically a while ago.} -% Samples (colored lines) of $\pr{z \gvn \ndphi}$ where each $\ndphi$ is drawn from the hyperprior distribution $\pr{\ndphi}$ given in \Eq{eqn:priorcov}.} -% \figlabel{fig:prior} -%\end{figure} -\que{Should I add back the figure of prior samples?} +%\que{Should I add back the figure of prior samples?} -The sampler is initialized with $W=100$ walkers each with a value chosen from a Gaussian distribution of identity covariance around a sample from the hyperprior distribution. +The sampler is initialized with $W=100$ walkers each with a value chosen from a Gaussian distribution of identity covariance around a sample from the hyperprior distribution. + +\section{Results} +\label{sec:alldata} + +Here, we compare the results of the \Chippr\ methodology with those of established \nz\ estimators under the three traditional measures of \pz\ uncertainty one at a time: +\Sect{sec:scatter} concerns the redshift-dependent intrinsic scatter, \Sect{sec:outliers} concerns realistically complex catastrophic outlier populations, and \Sect{sec:bias} concerns the canonical bias in the mean redshift. \subsection{Intrinsic scatter} \label{sec:scatter} -%Several factors contribute to photometric redshifts' intrinsic scatter. -%Distant galaxies are dimmer compared to galaxies of identical luminosity that are closer, driving up photometric errors in flux-limited surveys. -%The nature of the galaxy sample at higher redshifts also changes, meaning the generation of the photometric redshift posterior based on an a locally-calibrated SED template library or spectroscopically-confirmed training set is more likely to be inappropriate, leading to broader features. -%In general, the galaxies that could not have been observed spectroscopically will have different and noisier photo-$z$ likelihoods than those that could fall into a spectroscopic training set (or spectroscopically derived template library). -%This effect may be stronger for high-redshift galaxies. +%Several factors contribute to photometric redshifts' intrinsic scatter. +%Distant galaxies are dimmer compared to galaxies of identical luminosity that are closer, driving up photometric errors in flux-limited surveys. +%The nature of the galaxy sample at higher redshifts also changes, meaning the generation of the photometric redshift posterior based on an a locally-calibrated SED template library or spectroscopically-confirmed training set is more likely to be inappropriate, leading to broader features. +%In general, the galaxies that could not have been observed spectroscopically will have different and noisier photo-$z$ likelihoods than those that could fall into a spectroscopic training set (or spectroscopically derived template library). +%This effect may be stronger for high-redshift galaxies. \Fig{fig:pzs-scatter} shows some examples of \pzpdf s generated with only the systematic of intrinsic scatter, at the level of the \lsst\ requirements on the left and twice that on the right. One can see that the histogram of redshift estimates is broader than that of true redshifts, and that the effect is substantially more pronounced by just doubling the intrinsic scatter from the level of the \lsst\ requirements. \begin{figure*} \begin{center} - \includegraphics[width=0.45\textwidth]{figures/chippr/samplepzs_scatter1.png} - \includegraphics[width=0.45\textwidth]{figures/chippr/samplepzs_scatter2.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/single_varsigmas-mega_scatter.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_hivarsig-mega_scatter.png} \caption{ - Examples of mock \pzpdf s (colored lines) generated with intrinsic scatter at the \lsst\ requirements (left) and twice the \lsst\ requirements (right), including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored lines), the true redshifts of the example \pzpdf s (black vertical lines). - A histogram (gray) of points in each dimension is shown in the respective inset, with the true redshift distribution (blue curve) and implicit prior (red curve). - \aim{TODO: Enlarge axis labels and label panels. - Add watermark of ``mock data'' in UL corner, same for ``results of inference'' on other kind of plot.} + Examples of mock \pzpdf s generated with intrinsic scatter at the \lsst\ requirements (left) and twice the \lsst\ requirements (right), including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored step functions), and the true redshifts of the example \pzpdf s (colored vertical lines). + A histogram (light gray) of points in each dimension is shown in the respective inset, with the true redshift distribution (black) and implicit prior (dark gray). +% \aim{TODO: Label panels. +% Show the mean redshift for each estimator.} } \label{fig:pzs-scatter} \end{center} @@ -561,13 +594,13 @@ \subsection{Intrinsic scatter} \begin{figure*} \begin{center} - \includegraphics[width=0.45\textwidth]{figures/chippr/results_scatter1.png} - \includegraphics[width=0.45\textwidth]{figures/chippr/results_scatter2.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/single_varsigmas_log_estimators.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_hivarsig_log_estimators.png} \caption{ The results of \Chippr\ (samples in light blue and optimization in dark blue) and the alternative approaches (the stacked estimator in red and the histogram of modes in yellow) on \pzpdf s with intrinsic scatter of the \lsst\ requirements (left) and twice that (right), with the true redshift density (black curve) and implicit prior (gray curve). \Chippr\ is robust to intrinsic scatter, but the alternatives suffer from overly broad \nz\ estimates that worsen with increasing intrinsic scatter. - \aim{TODO: Label panels. - Add watermark of ``results of inference'' in UL corner, same for ``mock data'' on other kind of plot.} +% \aim{TODO: Label panels. +% Show the mean redshift for each estimator.} } \label{fig:results-scatter} \end{center} @@ -587,14 +620,16 @@ \subsection{Catastrophic outliers} \begin{figure} \begin{center} - \includegraphics[width=0.45\textwidth]{figures/chippr/single_uout_mega_scatter.png}\\ + \includegraphics[width=0.45\textwidth]{figures/chippr/single_uout-mega_scatter.png}\\ \includegraphics[width=0.45\textwidth]{figures/chippr/single_uout_log_estimators.png} \caption{ - Top: Examples of \pzpdf s with a uniform catastrophic outlier population at the level of the \lsst\ requirements, including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored curves), and the true redshifts of the example \pzpdf s (black vertical lines), with marginal histograms (gray) for each dimension with the true redshift distribution (blue curve) and implicit prior (red curve) in the insets. + Top: Examples of \pzpdf s with a uniformly distributed catastrophic outlier population at the level of the \lsst\ requirements, including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored step functions), and the true redshifts of the example \pzpdf s (colored vertical lines), with marginal histograms (light gray) for each dimension with the true redshift distribution (black) and implicit prior (dark gray) in the insets. +% \aim{TODO: Include slice further up. +% Label panels.} Bottom: The results of \Chippr\ (samples in light blue, optimization in dark blue) and the alternative approaches (the stacked estimator in red, the histogram of modes in yellow) on \pzpdf s with uniformly distributed catastrophic outliers, with the true redshift density (black curve) and implicit prior (gray curve). +% \aim{TODO: Label panels. +% Show the mean redshift for each estimator.} The presence of the catastrophic outlier population broadens the histogram of modes and stacked estimator of the redshift distribution, but the result of \Chippr\ is unbiased. - \aim{TODO: Enlarge axis labels on top and label panels.\\ - Also, add watermark of ``mock data'' in UL corner of top panel and ``results of inference'' on bottom panel.} } \label{fig:uniform-outliers} \end{center} @@ -613,12 +648,12 @@ \subsection{Catastrophic outliers} \begin{figure*} \begin{center} - \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_eout_mega_scatter.png} - \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_rout_mega_scatter.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_eout-mega_scatter.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_rout-mega_scatter.png} \caption{ - Examples of \pzpdf s with a catastrophic outlier population like that seen in template-fitting \pzpdf\ codes (left) and machine learning \pzpdf\ codes (right), including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored curves), and the true redshifts of the example \pzpdf s (black vertical lines), with marginal histograms (gray) for each dimension with the true redshift distribution (blue curve) and implicit prior (red curve) in the insets. - \aim{TODO: Enlarge axis labels and label panels. - Add watermark of ``mock data'' in UL corner, same for ``results of inference'' on other kind of plot.} + Examples of \pzpdf s with a catastrophic outlier population like that seen in template-fitting \pzpdf\ codes (left) and machine learning \pzpdf\ codes (right), including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored step functions), and the true redshifts of the example \pzpdf s (colored vertical lines), with marginal histograms (light gray) for each dimension with the true redshift distribution (black) and implicit prior (dark gray) in the insets. +% \aim{TODO: Label panels. +% Include slices higher up.} } \label{fig:nonuniform-outliers-data} \end{center} @@ -637,250 +672,275 @@ \subsection{Catastrophic outliers} \caption{ The results of \Chippr\ (samples in light blue and optimization in dark blue) and the alternative approaches (the stacked estimator in red, the histogram of modes in yellow) on \pzpdf s with catastrophic outliers like those seen in template-fitting \pzpdf\ codes (left) and machine learning \pzpdf\ codes (right) to the \lsst\ requirements, with the true redshift density (black curve) and implicit prior (gray curve). Though the histogram of modes is most sensitive to a catastrophic outlier population, the stacked estimator also overestimates \nz\ under (machine learning-like outliers) and beyond (template fitting-like outliers). - \aim{TODO: Label panels. - Add watermark of ``results of inference'' in UL corner, same for ``mock data'' on other kind of plot.} +% \aim{TODO: Label panels. +% Show the mean redshift for each estimator.} } \label{fig:nonuniform-outliers-results} \end{center} \end{figure*} -\subsection{Systematic bias} +\subsection{Canonical bias} \label{sec:bias} -Systematic bias in \pz\ point estimates is a concern for \lsst's cosmology results, for the same reasons explored in \citet{hoyle_dark_2017}. -However, in the context of \pzpdf s, the notion of redshift bias is a form of model misspecification. -Consider that if bias were included in the framework of Figure~\ref{fig:pedagogical_scatter}; -a simple linear transformation of $z_{\mathrm{phot}} \to z_{\mathrm{phot}} - \Delta_{z} (1 + z_{\mathrm{phot}})$ would eliminate the bias. -Regardless, for completeness, a test at ten times the bias of the \lsst\ requirements, with no redshift-dependent intrinsic scatter nor catastrophic outliers, is provided in \Fig{fig:bias}. +Systematic bias in \pz\ point estimates, is a concern for \lsst's cosmology results, for the same reasons explored in \citet{hoyle_dark_2018}. +This form of bias is typically summarized by a shift parameter $\Delta_{z} = (\langle \pr{z \gvn \hat{\ndphi}} \rangle - \langle \pr{z \gvn \ndphi^{\dagger}} \rangle)$ representing a difference between the first moment of the estimated redshift density function and that of the true redshift density function. +To distinguish other aforementioned manifestations of bias from this common form of bias, we refer to $\Delta_{z}$ as the \textit{canonical bias}. + +In the context of \pzpdf s, the canonical bias represents an instance of model misspecification. +Consider that if the canonical bias were included in the framework of Figure~\ref{fig:pedagogical_scatter}, it could be trivially modeled out as a simple linear transformation of $z_{\mathrm{phot}} \to z_{\mathrm{phot}} - \Delta_{z} (1 + z_{\mathrm{phot}})$ of the $(z_{\mathrm{spec}}, z_{\mathrm{phot}})$ space. +Regardless, for completeness, a test at ten times the canonical bias of the \lsst\ requirements, with no redshift-dependent intrinsic scatter nor catastrophic outliers, is provided in \Fig{fig:bias}. \begin{figure} \begin{center} - \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_neghivarbias_mega_scatter.png}\\ + \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_neghivarbias-mega_scatter.png}\\ \includegraphics[width=0.45\textwidth]{figures/chippr/thesis_neghivarbias_log_estimators.png} \caption{ - Left: Examples of \pzpdf s with ten times the bias of the \lsst\ requirements, including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored curves), and the true redshifts of the example \pzpdf s (black vertical lines), with marginal histograms (gray) for each dimension with the true redshift distribution (blue curve) and implicit prior (red curve) in the insets. - Right: The results of \Chippr\ (samples in light blue, optimization in dark blue) and the alternative approaches (the stacked estimator in red, the histogram of modes in yellow) on \pzpdf s with uniformly distributed catastrophic outliers, with the true redshift density (black curve) and implicit prior (gray curve). + Top: Examples of \pzpdf s with ten times the bias of the \lsst\ requirements, including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored step functions), and the true redshifts of the example \pzpdf s (colored vertical lines), with marginal histograms (light gray) for each dimension with the true redshift distribution (black) and implicit prior (dark gray) in the insets. +% \aim{TODO: Include slice further up. +% Label panels.} + Bottom: The results of \Chippr\ (samples in light blue, optimization in dark blue) and the alternative approaches (the stacked estimator in red, the histogram of modes in yellow) on \pzpdf s with ten times the bias of the \lsst\ requirements, with the true redshift density (black curve) and implicit prior (gray curve). +% \aim{TODO: Label panel. +% Show the mean redshift for each estimator.} The impact of bias at even ten times the level of the \lsst\ requirements is almost imperceptible on all estimators, though the \Chippr\ \mmle\ minimizes the information loss regardless. - \aim{TODO: Enlarge axis labels on top and label panels. - Also, add watermark of ``mock data'' in UL corner of top panel and ``results of inference'' on bottom panel.} } \label{fig:bias} \end{center} \end{figure} -As expected based on consistency with the forward model, \Chippr\ is completely resistant to bias, and the alternative estimators are only weakly affected, with information loss two and four times greater than that of the \Chippr\ \mmle\ for the histogram of modes and stacked estimator respectively. -\que{Did I effectively explain why \Chippr\ is expected to be unaffected by bias of this form?} - -\subsection{Implicit prior} -\label{sec:interim} - -\chippr\ can handle any implicit prior with support over the redshift range where \nz\ is defined, but some archetypes of implicit prior are more likely to be encountered in the wilds of \pzpdf\ codes. -Ideally, an uninformative implicit prior would be used, although it may be complicated to compute from the covariances of the raw data. -Template-fitting codes have an explicit prior input formed by redshifting a small number of templates, leading to a highly nonuniform but physically-motivated interim prior. -%Another potential method for selecting an interim prior with support over the entire redshift range expected of the photometric survey is to sum two or more $N(z)$ distributions obtained from reliable photometric surveys in the past. -%This is just as problematic as using a biased spectroscopically derived $N(z)$ as the interim prior because the sum of redshift distributions for two or more surveys does not reflect our beliefs about the true distribution for a single survey even though it provides support over the same redshift range. -%To simulate this case, we choose an interim prior with more weight at high and low redshifts than for mid-range redshifts. -Machine learning approaches tend to be trained on previously observed data sets that are biased towards low redshift, which biases the implicit prior towards low redshift. -% \aim{reference chapter 3 for complication of algorithm} -Some efforts have been made to modify an observationally informed implicit prior so that it is more representative of the photometric data for which redshifts are desired \citep{sheldon_photometric_2012}, but, unless it is equal to the true \nz, it will propagate to the results of traditional \nz\ estimation methods. -%Because low-redshift galaxies are more likely to be bright enough to be observed by such a survey, $N(z)$ determined from that sample may be heavily biased to low redshift galaxies. -%By contrast, the galaxies that were unobserved in such a survey are more likely be dimmer, making them more likely to be at higher redshifts. -%Since the interim prior is not compatible with our beliefs about the true redshift distribution, the resulting interim redshift posteriors will be inappropriate. - -\Fig{fig:pzs-priors} shows examples of \pzpdf s with a low-redshift favoring implicit prior emulating that of a machine learning approach to \pz\ estimation (left panel) and a more complex interim prior emulating that of a template-fitting \pz\ method (right panel). -One can see that the \pzpdf s take different shapes from one another even though the marginal histograms of the points are identical. -The machine learning-like implicit prior has been modified to have nonzero value at high-redshift because the implicit prior must be strictly positive definite for the \Chippr\ model to be valid. - -\begin{figure*} - \begin{center} - \includegraphics[width=0.45\textwidth]{figures/chippr/samplepzs_trpr.png} - \includegraphics[width=0.45\textwidth]{figures/chippr/samplepzs_tmpr.png} - \caption{ - Examples of mock \pzpdf s (colored lines) generated with a machine learning-like implicit prior (left) and a template-fitting-like implicit prior (right), including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored lines), the true redshifts of the example \pzpdf s (black vertical lines). - A histogram (gray) of points in each dimension is shown in the respective inset, with the true redshift distribution (blue curve) and implicit prior (red curve). - \aim{TODO: Enlarge axis labels. - Label panels. - Add watermark of ``mock data'' in UL corner, same for ``results of inference'' on other kind of plot.} - } - \label{fig:pzs-priors} - \end{center} -\end{figure*} - -\Fig{fig:results-priors} shows the performance of \Chippr\ and the traditional methods on \pzpdf s generated with nontrivial implicit priors. -In both cases, the \Chippr\ \mmle\ effectively recovers the true redshift distribution, and the distribution of \nz\ parameter values reflects higher uncertainty where the implicit prior undergoes large changes in derivative. -The alternatives, on the other hand, are biased by the implicit prior except where it is flat, in the case of high redshifts for the machine learning-like implicit prior, resulting in over $1,000$ times the information loss on \nz\ for the machine learning-like implicit prior and some $5-20$ times the information loss for the template fitting-like implicit prior, relative to the \Chippr\ \mmle. - -\begin{figure*} - \begin{center} - \includegraphics[width=0.45\textwidth]{figures/chippr/results_trpr.png} - \includegraphics[width=0.45\textwidth]{figures/chippr/results_tmpr.png} - \caption{ - The results of \Chippr\ (samples in light blue and optimization in dark blue) and the alternative approaches (the stacked estimator in red and the histogram of modes in yellow) on \pzpdf s with an implicit prior like that of machine learning \pzpdf\ approaches (left) and an implicit prior like that of template-fitting \pzpdf\ codes (right), with the true redshift density (black curve) and implicit prior (gray curve). - \Chippr\ is robust to a nontrivial implicit prior, but the alternatives are biased toward the implicit prior. - \aim{TODO: Label panels. - Add watermark of ``results of inference'' in UL corner, same for ``mock data'' on other kind of plot.} - } - \label{fig:results-priors} - \end{center} -\end{figure*} - -\que{Move \Sect{sec:violations} here?} - -The main implication of the response of \nz\ estimates to a nontrivial implicit prior is that the implicit prior must be accounted for when using \pzpdf\ catalogs. +As expected based on self-consistency of the forward-modeled \pzpdf s, \Chippr\ is immune to linear bias of the form of $\Delta_{z}$. +Furthermore, the alternative estimators are only weakly affected, with information loss two and four times greater than that of the \Chippr\ \mmle\ for the histogram of modes and stacked estimator respectively. +(This general robustness may suggest that the canonical bias may not be the most relevant measure of performance of estimators of \nz.) \section{Discussion} \label{sec:results} -The experiments of \Sect{sec:alldata} isolate the potential sources of error in \nz\ estimation one at a time. -Now, we stress-test \Chippr\ by investigating two realistically complex cases, one in which the \nz\ estimates are made tomographically as in a modern cosmological analysis (\Sect{sec:lsstdemo}) and one in which the \nz\ estimators are not provided with the same implicit prior used to generate the \pzpdf\ catalog (\Sect{sec:violations}). - -\subsection{LSST Requirements} -\label{sec:lsstdemo} - -It is of interest to explore the impact of incorrectly estimated \nz\ on the cosmological inference to answer the question of how wrong we will be in our understanding of the universe if we incorrectly constrain \nz. -To test the impact of these uncertainties, we simulate mock data with all three effects with which \lsst\ is concerned at the levels of Table~\ref{tab:lsstsrd} and propagate the results of \Chippr\ and the other estimators to a Fisher matrix forecast using \cosmolike\ \citep{krause_cosmolike_2017}, a publicly available cosmological forecasting code. -%Though redshift tomography is non-physical, as redshift is a continuous random variable, and binning in estimated redshift introduces poorly understood systematic error, we perform this analysis as an example of how it affects the current standard in how cosmological parameters are constrained by galaxy surveys, rather than how we think they ought to be constrained. -%\aim{Don't introduce anti-tomography rant here, kind of controversial and should be explored where it has its own space.} - -\begin{figure} - \begin{center} - \includegraphics[width=0.45\textwidth]{figures/chippr/cosmolike_inputs.png} - \caption{ - The \lsst-like tomographic binning and true redshift distribution, where the truth (solid) is a PDF evaluated on a fine grid of $350$ redshifts $0.0101 < z < 3.5001$, and the binned (dashed) and drawn (dotted) \nz\ are piecewise constant functions evaluated in $35$ evenly spaced bins, for four different tomographic bins (colors). - } - \label{fig:tomobins} - \end{center} -\end{figure} +The experiments of \Sect{sec:alldata} quantify the influence on each estimator of \nz\ due to each of the canonical types of \pz\ error one at a time in isolation. +Now, we stress-test \Chippr\ by exploring the impact of the implicit prior, which has thus far not received much attention in the literature. +%two realistically complex cases, one in which the \nz\ estimates are made tomographically as in a modern cosmological analysis (\Sect{sec:lsstdemo}) and one +\Sect{sec:interim} demonstrates the sensitivity of \nz\ estimation methods to realistically complex implicit priors, and \Sect{sec:violations} demonstrates the consequences of mischaracterization of the implicit prior used to generate the \pzip\ catalog. +These results provide compelling motivation for the \pz\ community to prioritize the study of implicit priors of existing and developing \pzpdf\ techniques. -\dwh{We consider as ground truth a set of known \nz\ corresponding to each of four hypothetical samples of galaxies and the corresponding cosmological parameter covariance matrix. -The \nz\ of each galaxy subsample emulates that anticipated of galaxies binned by a redshift point estimate, as is common in tomographic redshift analyses, though our experimental procedure is agnostic to how the samples are identified. -The cosmological parameter covariance matrices are those used for \desc\ forecasting with the ground truth \nz\ in the same four bins.} -The true \nz\ in each pre-defined bin is already provided in the form of an evaluation of a function on a fine grid of $350$ redshifts $0.0101 < z < 3.5001$. +%\que{Add back the results of the LSST requirements here?} -First, we bin them down to a piecewise constant parameterization with a manageable $35$ hyperparameters for \chippr's sampling capabilities. -Next, we draw $10^{4}$ true redshifts from the binned true \nz\ for each tomographic bin. -The original, binned, and drawn \nz\ are shown in \Fig{fig:tomobins}. -We emulate \pzpdf s for the $10^{4}$ true redshifts drawn from the true \nz\ in each bin using the procedure of \Fig{fig:flowchart} with all three effects of Table~\ref{tab:lsstsrd} -at their given levels. -Illustrations of this process are provided in \Fig{fig:per-bin-scatter}. +%\subsection{LSST Requirements} +%\label{sec:lsstdemo} +% +%It is of interest to explore the impact of incorrectly estimated \nz\ on the cosmological inference to answer the question of how wrong we will be in our understanding of the universe if we incorrectly constrain \nz. +%To test the impact of these uncertainties, we simulate mock data with all three effects with which \lsst\ is concerned at the levels of Table~\ref{tab:lsstsrd} and propagate the results of \Chippr\ and the other estimators to a Fisher matrix forecast using \cosmolike\ \citep{krause_cosmolike_2017}, a publicly available cosmological forecasting code. +% +%\begin{figure} +% \begin{center} +% \includegraphics[width=0.45\textwidth]{figures/chippr/cosmolike_inputs.png} +% \caption{ +% The \lsst-like tomographic binning and true redshift distribution, where the truth (solid) is a PDF evaluated on a fine grid of $350$ redshifts $0.0101 < z < 3.5001$, and the binned (dashed) and drawn (dotted) \nz\ are piecewise constant functions evaluated in $35$ evenly spaced bins, for four different tomographic bins (colors). +% } +% \label{fig:tomobins} +% \end{center} +%\end{figure} +% +%\dwh{We consider as ground truth a set of known \nz\ corresponding to each of four hypothetical samples of galaxies and the corresponding cosmological parameter covariance matrix. +%The \nz\ of each galaxy subsample emulates that anticipated of galaxies binned by a redshift point estimate, as is common in tomographic redshift analyses, though our experimental procedure is agnostic to how the samples are identified. +%The cosmological parameter covariance matrices are those used for \desc\ forecasting with the ground truth \nz\ in the same four bins.} +%The true \nz\ in each pre-defined bin is already provided in the form of an evaluation of a function on a fine grid of $350$ redshifts $0.0101 < z < 3.5001$. +% +%First, we bin them down to a piecewise constant parameterization with a manageable $35$ hyperparameters for \chippr's sampling capabilities. +%Next, we draw $10^{4}$ true redshifts from the binned true \nz\ for each tomographic bin. +%The original, binned, and drawn \nz\ are shown in \Fig{fig:tomobins}. +%We emulate \pzpdf s for the $10^{4}$ true redshifts drawn from the true \nz\ in each bin using the procedure of \Fig{fig:flowchart} with all three effects of Table~\ref{tab:lsstsrd} +%at their given levels. +%Illustrations of this process are provided in \Fig{fig:per-bin-scatter}. +% +%\begin{figure*} +% \begin{center} +% \includegraphics[width=0.24\textwidth]{figures/chippr/0single_lsst_mega_scatter.png} +% \includegraphics[width=0.24\textwidth]{figures/chippr/1single_lsst_mega_scatter.png} +% \includegraphics[width=0.24\textwidth]{figures/chippr/2single_lsst_mega_scatter.png} +% \includegraphics[width=0.24\textwidth]{figures/chippr/3single_lsst_mega_scatter.png} +% \caption{As in \Fig{fig:mega_scatter}, with a different tomographic bin in each panel and the three effects of intrinsic scatter, uniformly distributed catastrophic outliers, and bias at the levels of the \lsst\ SRD, given in Table~\ref{tab:lsstsrd}. +% \aim{TODO: Make this one big plot instead of four little ones to eliminate repeated insets and legend. +% Enlarge axis labels. +% Label panels. +% Also, add watermark of ``mock data'' in UL corner, same for ``results of inference'' on other kind of plot. +% Show the mean redshift for each estimator of N(z), cite the DES papers that motivate this.} +% } +% \label{fig:per-bin-scatter} +% \end{center} +%\end{figure*} +% +%\que{Is the distinction between binned samples determined by some observational property and probabilistic redshift distributions sufficiently clear?} +% +%We then make a point estimate of \nz\ using \chippr's \mmle\ optimization option as well as the alternative methods on the \pzpdf\ catalog for each tomographic bin, shown in \Fig{fig:per-bin-ests}, because \cosmolike\ produces cosmology constraints from a single \nz\ result, rather than samples from the full posterior probability density of possible \nz. +%Note that \Fig{fig:per-bin-ests} is shown in linear rather than log probability units, unlike all other plots in this paper, to better show the behavior at low probability. +%The excessive breadth of the alternative estimators can be seen quite plainly. +% +%\begin{figure*} +% \begin{center} +% \includegraphics[width=0.24\textwidth]{figures/chippr/0single_lsst_lin_estimators.png} +% \includegraphics[width=0.24\textwidth]{figures/chippr/1single_lsst_lin_estimators.png} +% \includegraphics[width=0.24\textwidth]{figures/chippr/2single_lsst_lin_estimators.png} +% \includegraphics[width=0.24\textwidth]{figures/chippr/3single_lsst_lin_estimators.png} +% \caption{ +% The \chippr-derived and other estimators of \nz\ in each tomographic bin, with the true \nz\ (black), the implicit prior (gray), stacked estimator (red), histogram of modes (yellow), and \Chippr\ \mmle\ (blue). +% The result of stacking is far too broad for \lsst-like \pzpdf s, even moreso than the simplistic histogram of modes. +% \aim{TODO: Make this one big plot instead of four little ones to eliminate repeated axis labels and legend. +% Label figure/panels. +% Also, add watermark of ``results of inference'' in UL corner, same for ``mock data'' on other kind of plot. +% Show the mean redshift for each estimator of N(z), cite the DES papers that motivate this.} +% } +% \label{fig:per-bin-ests} +% \end{center} +%\end{figure*} +% +%We then use the different estimators of \nz\ in a cosmological forecasting procedure with \cosmolike, constraining $\Omega_{m}$, $\Omega_{b}$, $w_{a}$, $w_{0}$, $n_{s}$, $S_{8}$, and $H_{0}$. +%Though there are also slight differences in the angle of the error ellipses, the most striking effect is the broadening of the contours under the alternative estimators relative to \Chippr, which are almost indistinguishable from those derived by using the true redshift distribution in each bin. +%The stacked estimator is significantly worse than the \Chippr\ \mmle\ for all parameters except $\Omega_{b}$ and $H_{0}$. +%Stacking, however, outperforms the histogram of modes for all parameters except $\Omega_{m}$ and $S_{8}$, for which their constraints are quite similar. +%Though the true values of the parameters themselves were not accessible with the Fisher matrix-based framework, we calculate the seven-dimensional KLD for the three \nz\ estimators relative to the constraints derived from the true \nz, showing that \Chippr\ preserves information $200-800$ times better than the alternatives, with the histogram of modes doing about four times better than stacking. +%\que{So a Fisher matrix analysis inherently can't provide the bias, because that requires data. +%Because we have no data, only posteriors conditioned on hypothetical data, this isn't possible. +%Do you think it's sufficient to provide the bias on the moments of \nz, since that's what everyone uses anyway?} +% +%\begin{figure*} +% \begin{center} +% \includegraphics[width=0.9\textwidth]{figures/chippr/final_plot.png} +% \caption{ +% \que{Are the contours any easier to see now?} +% The result of propagating the estimators of \nz\ by stacking (red), the histogram of modes (yellow), \Chippr\ (blue), and the true \nz\ (black) of \Fig{fig:per-bin-ests} to a subset of cosmological parameters. +% For all parameters considered, \Chippr\ yields contours no broader than those corresponding to the true \nz, whereas for most parameters, stacking and the histogram of modes yield broader contours. +% \aim{TODO: Fix formatting of axis labels. +% Standardize ticklabels.} +% } +% \label{fig:cornerplot} +% \end{center} +%\end{figure*} +% +%\que{Does this discussion adequately quantify how good \Nz\ has to be and how wrong we'll be if we estimate it wrong? +% Suggestions for how to better establish context would be appreciated.} -\begin{figure*} - \begin{center} - \includegraphics[width=0.24\textwidth]{figures/chippr/0single_lsst_mega_scatter.png} - \includegraphics[width=0.24\textwidth]{figures/chippr/1single_lsst_mega_scatter.png} - \includegraphics[width=0.24\textwidth]{figures/chippr/2single_lsst_mega_scatter.png} - \includegraphics[width=0.24\textwidth]{figures/chippr/3single_lsst_mega_scatter.png} - \caption{As in \Fig{fig:mega_scatter}, with a different tomographic bin in each panel and the three effects of intrinsic scatter, uniformly distributed catastrophic outliers, and bias at the levels of the \lsst\ SRD, given in Table~\ref{tab:lsstsrd}. - \aim{TODO: Make this one big plot instead of four little ones to eliminate repeated insets and legend. - Enlarge axis labels. - Label panels. - Also, add watermark of ``mock data'' in UL corner, same for ``results of inference'' on other kind of plot.} - } - \label{fig:per-bin-scatter} - \end{center} -\end{figure*} +\subsection{Realistically complex implicit prior} +\label{sec:interim} -\que{Is the distinction between binned samples determined by some observational property and probabilistic redshift distributions sufficiently clear?} +\chippr\ can handle any implicit prior with support over the redshift range where \nz\ is defined, but some archetypes of implicit prior are more likely to be encountered in the wilds of \pzip\ codes. +Ideally, an uninformative implicit prior would be used, although it may be complicated to compute from the covariances of the raw data. +Template-fitting codes have an explicit prior input formed by redshifting a small number of templates, leading to a highly nonuniform but physically-motivated interim prior. +%Another potential method for selecting an interim prior with support over the entire redshift range expected of the photometric survey is to sum two or more $N(z)$ distributions obtained from reliable photometric surveys in the past. +%This is just as problematic as using a biased spectroscopically derived $N(z)$ as the interim prior because the sum of redshift distributions for two or more surveys does not reflect our beliefs about the true distribution for a single survey even though it provides support over the same redshift range. +%To simulate this case, we choose an interim prior with more weight at high and low redshifts than for mid-range redshifts. +Machine learning approaches tend to be trained on previously observed data sets that are biased towards low redshift, which biases the implicit prior towards low redshift. +Some efforts have been made to modify an observationally informed implicit prior so that it is more representative of the photometric data for which redshifts are desired \citep{sheldon_photometric_2012}, but, unless it is equal to the true \nz, it will propagate to the results of traditional \nz\ estimation methods. +%Because low-redshift galaxies are more likely to be bright enough to be observed by such a survey, $N(z)$ determined from that sample may be heavily biased to low redshift galaxies. +%By contrast, the galaxies that were unobserved in such a survey are more likely be dimmer, making them more likely to be at higher redshifts. +%Since the interim prior is not compatible with our beliefs about the true redshift distribution, the resulting interim redshift posteriors will be inappropriate. -We then make a point estimate of \nz\ using \chippr's \mmle\ optimization option as well as the alternative methods on the \pzpdf\ catalog for each tomographic bin, shown in \Fig{fig:per-bin-ests}, because \cosmolike\ produces cosmology constraints from a single \nz\ result, rather than samples from the full posterior probability density of possible \nz. -Note that \Fig{fig:per-bin-ests} is shown in linear rather than log probability units, unlike all other plots in this paper, to better show the behavior at low probability. -The excessive breadth of the alternative estimators can be seen quite plainly. +\Fig{fig:pzs-priors} shows examples of \pzip s with a low-redshift favoring implicit prior emulating that of a machine learning approach to \pz\ estimation (left panel) and a more complex interim prior emulating that of a template-fitting \pz\ method (right panel). +One can see that the \pzip s take different shapes from one another even though the marginal histograms of the points are identical. +The machine learning-like implicit prior has been modified to have nonzero value at high-redshift because the implicit prior must be strictly positive definite for the \Chippr\ model to be valid. \begin{figure*} \begin{center} - \includegraphics[width=0.24\textwidth]{figures/chippr/0single_lsst_lin_estimators.png} - \includegraphics[width=0.24\textwidth]{figures/chippr/1single_lsst_lin_estimators.png} - \includegraphics[width=0.24\textwidth]{figures/chippr/2single_lsst_lin_estimators.png} - \includegraphics[width=0.24\textwidth]{figures/chippr/3single_lsst_lin_estimators.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/single_lsst_trpr-mega_scatter.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/single_lsst_tmpr-mega_scatter.png} \caption{ - The \chippr-derived and other estimators of \nz\ in each tomographic bin, with the true \nz\ (black), the implicit prior (gray), stacked estimator (red), histogram of modes (yellow), and \Chippr\ \mmle\ (blue). - The result of stacking is far too broad for \lsst-like \pzpdf s, even moreso than the simplistic histogram of modes. - \aim{TODO: Make this one big plot instead of four little ones to eliminate repeated axis labels and legend. - Label figure/panels. - Also, add watermark of ``results of inference'' in UL corner, same for ``mock data'' on other kind of plot.} + Examples of mock \pzip s generated with a machine learning-like implicit prior (left) and a template-fitting-like implicit prior (right), including samples from the probability space of true and observed redshift (black points), \pzip s (colored step functions), the true redshifts of the example \pzip s (colored vertical lines). + A histogram (light gray) of points in each dimension is shown in the respective inset, with the true redshift distribution (black) and implicit prior (dark gray). +% \aim{TODO: Label panels. +% Include slices higher up.} } - \label{fig:per-bin-ests} + \label{fig:pzs-priors} \end{center} \end{figure*} -We then use the different estimators of \nz\ in a cosmological forecasting procedure with \cosmolike, constraining $\Omega_{m}$, $\Omega_{b}$, $w_{a}$, $w_{0}$, $n_{s}$, $S_{8}$, and $H_{0}$. -Though there are also slight differences in the angle of the error ellipses, the most striking effect is the broadening of the contours under the alternative estimators relative to \Chippr, which are almost indistinguishable from those derived by using the true redshift distribution in each bin. -The stacked estimator is significantly worse than the \Chippr\ \mmle\ for all parameters except $\Omega_{b}$ and $H_{0}$. -Stacking, however, outperforms the histogram of modes for all parameters except $\Omega_{m}$ and $S_{8}$, for which their constraints are quite similar. -Though the true values of the parameters themselves were not accessible with the Fisher matrix-based framework, we calculate the seven-dimensional KLD for the three \nz\ estimators relative to the constraints derived from the true \nz, showing that \Chippr\ preserves information $200-800$ times better than the alternatives, with the histogram of modes doing about four times better than stacking. -%\aim{TODO: Ask Elisabeth how to get the bias here.} -\que{So a Fisher matrix analysis inherently can't provide the bias, because that requires data. -Because we have no data, only posteriors conditioned on hypothetical data, this isn't possible. -Do you think it's sufficient to provide the bias on the moments of \nz, since that's what everyone uses anyway?} +\Fig{fig:results-priors} shows the performance of \Chippr\ and the traditional methods on \pzip s generated with nontrivial implicit priors. +In both cases, the \Chippr\ \mmle\ effectively recovers the true redshift distribution, and the distribution of \nz\ parameter values reflects higher uncertainty where the implicit prior undergoes large changes in derivative. +The alternatives, on the other hand, are biased by the implicit prior except where it is flat, in the case of high redshifts for the machine learning-like implicit prior, resulting in over $1,000$ times the information loss on \nz\ for the machine learning-like implicit prior and some $5-20$ times the information loss for the template fitting-like implicit prior, relative to the \Chippr\ \mmle. \begin{figure*} \begin{center} - \includegraphics[width=0.9\textwidth]{figures/chippr/final_plot.png} - \caption{ - \que{Are the contours any easier to see now?} - The result of propagating the estimators of \nz\ by stacking (red), the histogram of modes (yellow), \Chippr\ (blue), and the true \nz\ (black) of \Fig{fig:per-bin-ests} to a subset of cosmological parameters. - For all parameters considered, \Chippr\ yields contours no broader than those corresponding to the true \nz, whereas for most parameters, stacking and the histogram of modes yield broader contours. - \aim{TODO: Fix formatting of axis labels. - Standardize ticklabels.} + \includegraphics[width=0.45\textwidth]{figures/chippr/single_lsst_trpr_log_estimators.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/single_lsst_tmpr_log_estimators.png} + \caption + {The results of \Chippr\ (samples in light blue and optimization in dark blue) and the alternative approaches (the stacked estimator in red and the histogram of modes in yellow) on \pzip s with an implicit prior like that of machine learning \pzip\ approaches (left) and an implicit prior like that of template-fitting \pzip\ codes (right), with the true redshift density (black curve) and implicit prior (gray curve). + \Chippr\ is robust to a nontrivial implicit prior, but the alternatives are biased toward the implicit prior. +% \aim{TODO: Label panels. +% Show the mean redshift for each estimator.} } - \label{fig:cornerplot} + \label{fig:results-priors} \end{center} \end{figure*} -\que{Does this discussion adequately quantify how good \Nz\ has to be and how wrong we'll be if we estimate it wrong? - Suggestions for how to better establish context would be appreciated.} +The main implication of the response of \nz\ estimates to a nontrivial implicit prior is that the implicit prior must be accounted for when using \pzip\ catalogs. \subsection{Violations of the model} \label{sec:violations} -\que{Move \Sect{sec:interim} here?} - -In this test, without tomographic binning, the \pzip s are made to the \lsst\ requirements but the implicit prior used for the inference is not the same as the implicit prior used for generating the data. +In this test, the \pzip s are made to the \lsst\ requirements but the implicit prior used for the inference is not the same as the implicit prior used for generating the data. \Pzpdf\ codes do not generally provide their implicit prior, with the exception of some template-fitting techniques for which it is a known input. -If we naively used the \pzpdf\ catalog produced by a generic machine learning code and assumed a flat implicit prior, we would observe the contents of \Fig{fig:mischaracterized}. +If we naively used the \pzip\ catalog produced by a generic machine learning or template-fitting code and assumed a flat implicit prior, we would observe the contents of \Fig{fig:mischaracterized}. -\begin{figure} +\begin{figure*} \begin{center} -% \includegraphics[width=0.45\textwidth]{figures/chippr/misspecified_mega_scatter.png}\\ -% \includegraphics[width=0.45\textwidth]{figures/chippr/misspecified_log_estimators.png} - \caption{ - Top: Examples of \pzpdf s with systematics at the level of the \lsst\ requirements with a machine learning-like implicit prior, including samples from the probability space of true and observed redshift (black points), \pzpdf s (colored curves), and the true redshifts of the example \pzpdf s (black vertical lines), with marginal histograms (gray) for each dimension with the true redshift distribution (blue curve) and implicit prior (red curve) in the insets. - Bottom: The results of \Chippr\ (samples in light blue, optimization in dark blue) and the alternative approaches (the stacked estimator in red, the histogram of modes in yellow) on \pzpdf s with uniformly distributed catastrophic outliers, with the true redshift density (black curve) and the uniform implicit prior given to \chippr\ (gray curve). + \includegraphics[width=0.45\textwidth]{figures/chippr/single_lsst_trpr_wrong_log_estimators.png} + \includegraphics[width=0.45\textwidth]{figures/chippr/single_lsst_tmpr_wrong_log_estimators.png} + \caption + {The results of \Chippr\ (samples in light blue, optimization in dark blue) and the alternative approaches (the stacked estimator in red, the histogram of modes in yellow) when run with an incorrectly specified implicit prior (gray curve). + The data upon which each panel's results are based are provided in Figure~\ref{fig:pzs-priors}, where the left corresponds to the sort of implicit prior anticipated of machine learning approaches and the right corresponds to an implicit prior like that of a template-fitting code. + Here, \Chippr\ has been provided with a uniform implicit prior rather than those used to produce the mock \pzip s, and its performance is notably worse than when it is provided an accurate implicit prior, as in Figure~\ref{fig:results-priors}. +% \aim{Label panels. +% Show the mean redshift for each estimator.} When the incorrect implicit prior is provided to \chippr, even Bayesian inference cannot recover the true \nz. - \aim{TODO: Enlarge axis labels on top. - Label panels/figure. - Also, add watermark of ``mock data'' in UL corner of top panel and ``results of inference'' on bottom panel.} } \label{fig:mischaracterized} \end{center} -\end{figure} +\end{figure*} The results of using a mischaracterized implicit prior are disastrous, causing every estimator, including \Chippr, to be strongly biased. The stacked estimator and histogram of modes don't make use of the implicit prior so do no worse than when the implicit prior is accurately provided, but \Chippr\ is sensitive to prior misspecification, which violates the model upon which it is based. -It is thus crucial that \pzpdf\ methods always characterize and provide the implicit prior. +It is thus crucial that \pzip\ methods always characterize and provide the implicit prior. % removed ancient investigation on real data, would be hard to redo with the new \chippr\ code on this timescale given that I haven't even looked at the data format in 4 years \section{Conclusion} \label{sec:con} -This study derives and demonstrates a mathematically consistent inference of a one-point statistic, the redshift density function \nz, based on an arbitrary catalog of \pzpdf s. -The fully Bayesian method, based in the fundamental laws of probability, begins with a probabilistic graphical model corresponding to equations for the full posterior distribution over the parameters for \nz. -The method is validated on mock data and tested in the regime of \lsst\ with promising results, outperforming the traditional stacking estimator at the level of \nz\ as well as in terms of constraining power on the cosmological parameters. -Not only is this the only mathematically correct approach to the problem, it also recovers the true parameter values better than popular alternatives, as measured by the loss of information in \nz\ and the size of error ellipses in the space of cosmological parameters. +%\que{TODO: Break up conclusion into subsections of \Sect{sec:alldata}/\Sect{sec:results}?} -%In the tests on simulated data performed here, the full posterior distribution over the hyperparameters defining $N(z)$ derived by this method is consistent with the true redshift distribution function, making the mean of sampled values an excellent point estimator of $N(z)$. -%The information contained in the full posterior distribution's shape convey the traditional error bar information without having to explicitly propagate any error estimates. -\aim{TODO: Refer to quantitative results on KLD of \Nz\ and cosmological parameter space.} -%The results of those tests is summarized below and in \Tab{tab:kld}, where lower values indicate a closer match between the true $N(z)$ and the estimator. -%Tests were also performed on subsets of BOSS DR10 data with results consistent with those of simulations. +This study derives and demonstrates a mathematically consistent inference of a one-point statistic, the redshift density function \nz, based on an arbitrary catalog of \pzpdf s. +The fully Bayesian \Chippr\ model, based in the fundamental laws of probability, begins with a probabilistic graphical model corresponding to equations for the full posterior distribution over the parameters for \nz. +The \Chippr\ model is implemented in the publicly available \chippr\ code. +The method is implemented in the publicly available \chippr\ code and validated on mock data. +% at the level of \nz\ as well as in terms of constraining power on the cosmological parameters. -\dwh{\Chippr\ outperforms traditional estimators of \nz\ under the canonical \pz\ systematics of intrinsic scatter, catastrophic outliers, and bias. -\Chippr\ is also robust to nontrivial implicit priors corresponding to the specifics of the architecture of the method by which \pzpdf s are obtained. -However, \Chippr\ absolutely requires that the implicit prior is accurately known; using an implicit prior other than that which contributed to the production of the \pzpdf\ catalog results in catastrophically incorrect inference of \nz. -It is therefore imperative that those developing codes to obtain \pzpdf s provide a way to isolate the implicit prior and that those publishing \pzpdf\ catalogs provide the implicit prior to users. +%In the tests on simulated data performed here, the full posterior distribution over the hyperparameters defining $N(z)$ derived by this method is consistent with the true redshift distribution function, making the mean of sampled values an excellent point estimator of $N(z)$. +%The information contained in the full posterior distribution's shape convey the traditional error bar information without having to explicitly propagate any error estimates. +%The results of those tests is summarized below and in \Tab{tab:kld}, where lower values indicate a closer match between the true $N(z)$ and the estimator. +%Tests were also performed on subsets of BOSS DR10 data with results consistent with those of simulations. +Using a flexible, self-consistent forward model of the relationship between true and estimated redshifts, capable of encapsulating the complexity of observed redshift-photometry relations (e.g. \Fig{fig:pedagogical_scatter}), we emulate the canonical \pz\ error statistics, intrinsic scatter (\Sect{sec:scatter}), catastrophic outliers (\Sect{sec:outliers}), and canonical bias (\Sect{sec:bias}) one at a time. +Though these test cases may appear overly simplistic, they enable rigorous quantification of the relative performance of each \nz\ estimation techniques under the controlled conditions of each type of error in isolation, at levels equal to and beyond those of \lsst. +%\aim{TODO: point out that Fig 1 is uglier than 4, 5, 7, 8, 10, 11; these tests are simplistic, reality is more complex, but because \Chippr is provably correct, it can handle complexity of the real thing if p(z) is correct and captures complexity} + +Based on our tests, the following statements about the \Chippr\ methodology may be made with confidence: +\begin{itemize} +\item \Chippr\ outperforms traditional estimators of \nz\ under realistically complex conditions, even at pessimistic levels relative to future survey requirements on the traditional \pz\ error statistics, as demonstrated both by eye and according to KLD values corresponding to $10\%$ the information loss of alternative methods. +%\aim{TODO: Refer to quantitative results on KLD of \Nz.} +\item Both the \Chippr\ \mmle\ and the mean of \chippr\ samples are good point estimators of \nz, whereas the histogram of modes is very sensitive to outliers and the stacked estimator is always excessively broad. +\item The error bars on the posterior distribution over \nz\ hyperparameters are interpretable and arise naturally under \Chippr, unlike those that may be assumed for the conventional point estimators. +\end{itemize} +Not only is \Chippr\ the only mathematically correct approach to the problem, it also recovers the true values of the hyperparameters defining \nz\ better than popular alternatives, as measured by the loss of information in \nz. +% \ and the size of error ellipses in the space of cosmological parameters. +However, the mathematically valid approach to inference with probabilistic data products incurs nontrivial computational expense, motivating future work to optimize the implementation. + +Additionally, this work highlights a crucial and almost entirely overlooked complication to the usage of \pzpdf s, namely the implicit prior, motivating the following recommendations: +\begin{itemize} +\item In the presence of a nontrivial implicit prior corresponding to the specifics of the architecture of the method by which \pzpdf s are obtained, established methods cannot recover \nz; +a principled hierarchical inference such as \Chippr\ is the only way to recover \nz\ from \pzpdf s. +\item %\Chippr, however, is sensitive to misspecification of the implicit prior; +Neither \Chippr\ nor traditional alternatives can recover \nz\ in the presence of a misspecified implicit prior; +the implicit prior used to produce the \pzpdf\ catalog must be known and provided to \Chippr\ in order to recover the true \nz. +\end{itemize} +Given the significance of the implicit prior \citep{schmidt_evaluation_2020}, it is therefore imperative that those developing codes to obtain \pzpdf s provide a way to isolate the implicit prior and that those publishing \pzpdf\ catalogs provide the implicit prior to users. This mandate is easier said than done, both for template fitting and machine learning approaches. + While the implicit prior is often an explicit input to model-based routines, it may be defined in a space of redshift and SED templates. In this case, it may not be possible to apply \Chippr\ without marginalizing over additional variables $\psi$ for the SEDs. In other words, obtaining the implicit prior from a template fitting code may be challenging or even require consideration of higher-dimensional PDFs such as $\pr{z, \mathrm{SED} \gvn \psi^{*}}$. @@ -890,55 +950,68 @@ \section{Conclusion} Additionally, the weights may be stochastic, depending on the random seed used to initialize non-deterministic methods, precluding reproducibility. It is thus unclear whether the implicit prior can be meaningfully obtained from such methods at all. -\aim{TODO: new paragraph for what can go wrong, call to community for what to do about it, what aspects of implicit prior are knowable and not knowable? - if testable how so? outside the scope of paper, data producers be warned! - a likelihood is better -- give us that if you can! - focus on methods that acknowledge probabilistic structure of problem} +%\aim{TODO: new paragraph for what can go wrong, +% call to community for what to do about it, +% what aspects of implicit prior are knowable and not knowable? +% if testable how so? outside the scope of paper, data producers be warned! +% a likelihood is better -- give us that if you can! +% focus on methods that acknowledge probabilistic structure of problem.} A thorough investigation of the degree to which the implicit prior can be meaningfully obtained is outside this paper but should be a priority for all consumers of \pzpdf s. As an alternative, however, we must point out that if likelihoods were available rather than posteriors, the trouble with the implicit prior would be avoided altogether. -We thus encourage the community of those making \pzpdf s to consider developing such methods so that the resulting data products may be correctly used in scientific inference more generically.} +We thus encourage the community of those making \pzpdf s to consider developing such methods so that the resulting data products may be correctly used in scientific inference more generically. -\aim{TODO: claim that implications for tomographic binning are severe, if that can be motivated by Section~\ref{sec:lsstdemo}.} +%\aim{TODO: Claim that implications for tomographic binning are severe, if that can be motivated by mean \Nz shifts.} +% by Section~\ref{sec:lsstdemo}.} -The following conclusions and recommendations can be made with confidence: +%\subsection*{Recommendations for future work} -\begin{enumerate} - \item Both the \Chippr\ \mmle\ and the mean of \chippr\ samples are good point estimators of \nz, whereas the histogram of modes is very sensitive to outliers and the stacked estimator is always excessively broad. - \item The error bars on the posterior distribution over \nz\ hyperparameters are interpretable and arise naturally under \Chippr, unlike those that may be assumed for the conventional point estimators. - % \item The marginalized maximum likelihood estimator is an excellent estimator for strongly featured redshift distribution function with simple, clean photo-$z$ posteriors; stacking smooths features more than sampling and photo-$z$ point estimation. - \item When the implicit prior is known to be a poor match to the data, only the results of \Chippr\ are satisfactory estimators of the redshift distribution function because they are the only methods that can account for the bias induced on the \pzpdf\ catalog by the method that produces it; this is the most compelling case for the sampler because of the ubiquity of inappropriate interim priors. -\end{enumerate} +%The following conclusions and recommendations can be made with confidence: + +%\begin{enumerate} +% +% % \item The marginalized maximum likelihood estimator is an excellent estimator for strongly featured redshift distribution function with simple, clean photo-$z$ posteriors; stacking smooths features more than sampling and photo-$z$ point estimation. +% \item When the implicit prior is known to be a poor match to the data, only the results of \Chippr\ are satisfactory estimators of the redshift distribution function because they are the only methods that can account for the bias induced on the \pzpdf\ catalog by the method that produces it; this is the most compelling case for the sampler because of the ubiquity of inappropriate interim priors. +%\end{enumerate} -By showing that \Chippr\ is effective in recovering the true redshift distribution function and posterior distributions on its parameters from catalogs of \pzpdf s, this work supports the production of \pzpdf s by upcoming photometric surveys such as \lsst\ to enable more accurate inference of the cosmological parameters. -We discourage researchers from co-adding \pzpdf s or converting them into point estimates of redshift and instead recommend the use of Bayesian probability to guide the usage of \pzpdf s. -We emphasize to those who produce \pzpdf s from data that it is essential to release the implicit prior used in generating this data product in order for proper inference to be conducted by consumers of this information. -\aim{TODO: Strengthen language and get specific about what \pzpdf\ producers must do.} +By showing that \Chippr\ is effective in recovering the true redshift distribution function and posterior distributions on its parameters from catalogs of \pzpdf s, this work supports the production of \pzpdf s by upcoming photometric surveys such as \lsst\ to enable more accurate inference of the cosmological parameters. +We discourage researchers from co-adding \pzpdf s or converting them into point estimates of redshift and instead recommend the use of Bayesian probability to guide the usage of \pzpdf s. +We emphasize to those who produce \pzpdf s from data that it is essential to release the implicit prior used in generating this data product in order for any valid inference to be conducted by consumers of this information. +Methodologies for obtaining \pzpdf s must therefore be designed such that there is a known implicit prior, i.e. one that is not implicit at all, so that likelihoods may be recovered. -The technique herein developed is applicable with minimal modification to other one-point statistics of redshift to which we will apply this method in the future, such as the redshift-dependent luminosity function and weak lensing mean distance ratio. +The technique herein developed is applicable with minimal modification to other one-point statistics of redshift to which we will apply this method in the future, such as the redshift-dependent luminosity function and weak lensing mean distance ratio. Future work will also include the extension of this fully probabilistic approach to higher-order statistics of redshift such as the two-point correlation function. +\begin{acknowledgements} + AIM acknowledges support from the Max Planck Society and the Alexander von Humboldt Foundation in the framework of the Max Planck-Humboldt Research Award endowed by the Federal Ministry of Education and Research. + During the completion of this work, AIM was supported by National Science Foundation grant AST-1517237 and the U.S. Department of Energy, Office of Science, Office of Workforce Development for Teachers and Scientists, Office of Science Graduate Student Research (SCGSR) program, administered by the Oak Ridge Institute for Science and Education for the DOE under contract number DE‐SC0014664. + The authors thank Phil Marshall for advice on relevant examples, Elisabeth Krause for assistance with the \cosmolike\ code, Mohammadjavad Vakili for statistical insights, Geoffrey Ryan for programming advice, and Boris Leistedt for other helpful comments in the development of \Chippr. +% The authors also acknowledge +% \aim{TODO: Circulate draft to GCCL; submit to ApJ; circulate to Dan Foreman-Mackey, Boris Leistedt, Kate Storey-Fisher; post to arXiv; circulate to Johann Cohen-Tanugi, Will Hartley, Alan Heavens, Mike Jarvis, Francois Lanusse, Ann Lee, Rachel Mandelbaum, Phil Marshall, Chris Morrison, Jeff Newman, Sam Schmidt, Anze Slosar, Josh Speagle, others for feedback.} + This work was completed with generous nutritional support from the Center for Computational Astrophysics. +% \aim{TODO: Thank thesis readers.} +\end{acknowledgements} + +%\aim{TODO: add software citation section.} + \appendix %\renewcommand{\thesection}{\Alph{section}} %\renewcommand{\thesubsection}{\Alph{subsection}} - -\aim{TODO: Figure out how to get the appendices enumerated for reference earlier.} +\numberwithin{equation}{section} \section{Derivation} \label{app:math} -\aim{TODO: make notation consistent with the rest of the paper.} - -%We begin by parametrizing $N(z)$ in terms of $\vec{\theta}$, comprising some set of hyperparameters that define the form $N(z)$ may take in whatever basis we choose. -%We define a function $f_{\vec{\theta}}(z)=N(z)$ that transforms these hyperparameters into the redshift distribution function $N(z)$. -%Because +%We begin by parametrizing $N(z)$ in terms of $\vec{\theta}$, comprising some set of hyperparameters that define the form $N(z)$ may take in whatever basis we choose. +%We define a function $f_{\vec{\theta}}(z)=N(z)$ that transforms these hyperparameters into the redshift distribution function $N(z)$. +%Because %\begin{equation} %\eqlabel{eq:definition} %N(z) \propto p(z \gvn \vec{\theta}), %\end{equation} %we may discontinue discussion of $N(z)$ in favor of the likelihood $p(z|\vec{\theta})$. -In this paper, we work exclusively with log-probabilities. +We perform the derivation of \Eq{eqn:fullpost} using log-probabilities. What we wish to estimate is then the full log-posterior probability distribution (hereafter the full log-posterior) of the hyperparameters $\ndphi$ given the catalog of photometry $\{\data_{j}\}$. By Bayes' Rule, the full log-posterior @@ -949,30 +1022,30 @@ \section{Derivation} may be expressed in terms of the full log-likelihood probability distribution (hereafter the full log-likelihood) $\ln[\pr{\{\data_{j}\} \gvn \ndphi}]$ by way of a hyperprior log-probability distribution (hereafter the hyperprior) $\ln[\pr{\ndphi}]$ over the hyperparameters and the log-evidence probability of the data $\ln[\pr{\{\data_{j}\}}]$. However, the evidence is rarely known, so we probe the full log-posterior modulo an unknown constant of proportionality. -The full log-likelihood may be expanded in terms of a marginalization over the redshifts as parameters, as in +The full log-likelihood may be expanded in terms of a marginalization over the redshifts as parameters, as in \begin{equation} \label{eqn:marginalize} \ln[\pr{\{\data_{j}\} \gvn \ndphi}] = \ln\left[\integral{\pr{\{\data_{j}\} \gvn \{z_{j}\}} \pr{\{z_{j}\} \gvn \ndphi}}{\{z_{j}\}}\right]. \end{equation} -We shall make two assumptions of independence in order to make the problem tractable; their limitations are be discussed below. -First, we take $\ln[\pr{\{\data_{j}\} \gvn \{z_{j}\}}]$ to be the sum of $J$ individual log-likelihood distribution functions $\ln[\pr{\data_{j} \gvn z_{j}}]$, as in +We shall make two assumptions of independence in order to make the problem tractable; their limitations are be discussed below. +First, we take $\ln[\pr{\{\data_{j}\} \gvn \{z_{j}\}}]$ to be the sum of $J$ individual log-likelihood distribution functions $\ln[\pr{\data_{j} \gvn z_{j}}]$, as in \begin{equation} \label{eqn:indiedat} \ln[\pr{\{\data_{j}\} \gvn \{z_{j}\}}] = \sum_{j=1}^{J}\ \ln[\pr{\data_{j} \gvn z_{j}}], \end{equation} a result of the definition of probabilistic independence encoded by the box in \Fig{fig:pgm}. -Second, we shall assume the true redshifts $\{z_{j}\}$ are $J$ independent draws from the true $\pr{z \gvn \ndphi}$. -Additionally, $J$ itself is a Poisson random variable. -The combination of these assumptions is given by +Second, we shall assume the true redshifts $\{z_{j}\}$ are $J$ independent draws from the true $\pr{z \gvn \ndphi}$. +Additionally, $J$ itself is a Poisson random variable. +The combination of these assumptions is given by \begin{equation} \label{eqn:indie} \ln[\pr{\{z_{j}\} \gvn \ndphi}] = -\integral{f(z; \ndphi)}{z} + \sum_{j=1}^{J}\ \ln[\pr{z_{j} \gvn \ndphi}]. \end{equation} -%It is important to note that the integral $\integral{n(z)}{z} N(z)\ dz$ is not constrained to equal the variable defining the Poisson distribution but instead $J$ by \Eq{eq:definition}, which can be thought of as another parameter. +%It is important to note that the integral $\integral{n(z)}{z} N(z)\ dz$ is not constrained to equal the variable defining the Poisson distribution but instead $J$ by \Eq{eq:definition}, which can be thought of as another parameter. The derivation differs when $J$ is not known, say, when we want to learn about a distribution in nature rather than a distribution specific to data in hand, but for a photometric galaxy catalog where the desired quantity is $n(z)$ for the galaxies entering a larger cosmology calculation, it is a fixed quantity. -A detailed discussion of this matter may be found in \citet{foreman-mackey_exoplanet_2014}. -Applying Bayes' Rule, we may combine terms to obtain +A detailed discussion of this matter may be found in \citet{foreman-mackey_exoplanet_2014}. +Applying Bayes' Rule, we may combine terms to obtain \begin{align} \begin{split} \label{eqn:posterior} @@ -980,16 +1053,16 @@ \section{Derivation} \end{split} \end{align} -%\Eq{eq:posterior} contains two quantities that merit further discussion, the prior distribution $p(\vec{\theta})$ discussed further in \Sect{sec:exp} and the photo-$z$ log-likelihoods $\ln[p(\vec{d}_{j}|z_{j})]$ that have not been mentioned since \Eq{eq:marginalize}. -%Though photo-$z$ log-likelihoods would be desirable for use in these equations, they are not generally the product of either empirical and data-driven methods for obtaining photo-$z$ probability distributions. -%Though probabilistic photo-$z$s are typically reported as generic probability distributions $p(z_{j})$, the methods that produce them may be understood to always yield posteriors, probability distributions conditioned on the data we believe to be true. +%\Eq{eq:posterior} contains two quantities that merit further discussion, the prior distribution $p(\vec{\theta})$ discussed further in \Sect{sec:exp} and the photo-$z$ log-likelihoods $\ln[p(\vec{d}_{j}|z_{j})]$ that have not been mentioned since \Eq{eq:marginalize}. +%Though photo-$z$ log-likelihoods would be desirable for use in these equations, they are not generally the product of either empirical and data-driven methods for obtaining photo-$z$ probability distributions. +%Though probabilistic photo-$z$s are typically reported as generic probability distributions $p(z_{j})$, the methods that produce them may be understood to always yield posteriors, probability distributions conditioned on the data we believe to be true. %If they were not based in this assumption, they would require a sum over an infinite space of possible datasets. -Since we only have access to implicit \pz\ posteriors, we must be able to write the full log-posterior in terms of implicit \pz\ log-posteriors rather than the log-likelihoods of \Eq{eqn:posterior}. -To do so, we will need an explicit statement of this implicit prior $\ndphi^{*}$ for whatever method is chosen to produce the implicit \pz\ posteriors. +Since we only have access to \pzip s, we must be able to write the full log-posterior in terms of log \pzip s rather than the log-likelihoods of \Eq{eqn:posterior}. +To do so, we will need an explicit statement of this implicit prior $\ndphi^{*}$ for whatever method is chosen to produce the \pzip s. -To perform the necessary transformation from likelihoods to posteriors, we follow the reasoning of \citet{foreman-mackey_exoplanet_2014}. -Let us consider the probability of the parameters conditioned on the data and an interim prior and rewrite the problematic likelihood of \Eq{eqn:posterior} as +To perform the necessary transformation from likelihoods to posteriors, we follow the reasoning of \citet{foreman-mackey_exoplanet_2014}. +Let us consider the probability of the parameters conditioned on the data and an interim prior and rewrite the problematic likelihood of \Eq{eqn:posterior} as \begin{align} \label{eqn:trick} \begin{split} @@ -997,14 +1070,14 @@ \section{Derivation} \end{split} \end{align} -Once the implicit prior $\ndphi^{*}$ is explicitly introduced, we may expand the last term in \Eq{eqn:trick} according to Bayes' Rule to get +Once the implicit prior $\ndphi^{*}$ is explicitly introduced, we may expand the last term in \Eq{eqn:trick} according to Bayes' Rule to get \begin{align} \begin{split} \label{eqn:expand} \ln[\pr{\data_{j} \gvn z}] = & \ln[\pr{\data_{j} \gvn z}] + \ln[\pr{z \gvn \data_{j}, \ndphi^{*}}] + \ln[\pr{\data_{j} \gvn \ndphi^{*}}] - \ln[\pr{z \gvn \ndphi^{*}}] - \ln[\pr{\data_{j} \gvn z, \ndphi^{*}}]. \end{split} \end{align} -Because there is no direct dependence of the data upon the hyperparameters, we may again expand the term $\ln[\pr{\data_{j} \gvn z, \ndphi^{*}}]$ to obtain +Because there is no direct dependence of the data upon the hyperparameters, we may again expand the term $\ln[\pr{\data_{j} \gvn z, \ndphi^{*}}]$ to obtain \begin{align} \begin{split} \label{eqn:indterm} @@ -1016,21 +1089,22 @@ \section{Derivation} \label{eqn:cancel} \ln[\pr{\data_{j} \gvn z}] = \ln[\pr{z \gvn \data_{j}, \ndphi^{*}}] - \ln[\pr{z \gvn \ndphi^{*}}]. \end{equation} -We put this all together to get the full log-posterior probability distribution of +We put this all together to get the full log-posterior probability distribution of \begin{align} \begin{split} \label{eqn:final} \ln[\pr{\ndphi \gvn \{\data_{j}\}}] \propto & \ln[\pr{\ndphi}] + \ln \left[\integral{\exp \left[\sum_{j=1}^{J} \left(\ln[\pr{z \gvn \data_{j}, \ndphi^{*}}] + \ln[\pr{z \gvn \ndphi}] - \ln[\pr{z \gvn \ndphi^{*}}] \right)\right]}{z}\right] , \end{split} \end{align} +which is equivalent to that of \citet{hogg_inferring_2010}, though the context differs. -The argument of the integral in the log-posterior of \Eq{eqn:final} depends solely on knowable quantities (and those we must explicitly assume) and can be calculated for a given sample of \pz\ log-posteriors $\{\ln[\pr{z \gvn \data_{j}, \ndphi^{*}}]\}$ and the implicit prior $\pr{z \gvn \ndphi^{*}}$ with which they were obtained, noting the relation of +The argument of the integral in the log-posterior of \Eq{eqn:final} depends solely on knowable quantities (and those we must explicitly assume) and can be calculated for a given sample of log \pzip s $\{\ln[\pr{z \gvn \data_{j}, \ndphi^{*}}]\}$ and the implicit prior $\pr{z \gvn \ndphi^{*}}$ with which they were obtained, noting the relation of \begin{equation} \label{eqn:params} \pr{z \gvn \ndphi} = \frac{f(z; \ndphi)}{\integral{f(z; \ndphi)}{z}}. \end{equation} -Since we cannot know constant of proportionality, we sample the desired full log-posterior $\ln[\pr{\ndphi \gvn \{\data_{j}\}}]$ using Monte Carlo-Markov chain (MCMC) methods. -The method outlined here is valid regardless of how the implicit \pz\ log-posteriors are obtained so the many approaches to producing \pzpdf s will not be rehashed; though the matter is outside the scope of this paper, reviews of various methods have been presented in the literature \citep{sheldon_photometric_2012, ball_robust_2008, carrasco_kind_tpz:_2013, carrasco_kind_exhausting_2014}, and will be briefly reviewed in Schmidt, Malz \& Soo, et al. (in prep). +Since we cannot know constant of proportionality, we sample the desired full log-posterior $\ln[\pr{\ndphi \gvn \{\data_{j}\}}]$ using Monte Carlo-Markov chain (MCMC) methods. + % %\begin{align} %\begin{split} @@ -1039,43 +1113,39 @@ \section{Derivation} %\end{split} %\end{align} -\section{Convergence Criteria} -\label{app:acorr} - -In addition to qualitative visual inspection of the chains, two quantities that probe the convergence of the sampler are used in this study, the autocorrelation time and the Gelman-Rubin convergence criterion. -%\Fig{fig:chains} shows the %evolution of the values of one parameter of one walker over the course of all %iterations of the sampler. - -%\begin{figure} -%%\includegraphics[width=0.5\textwidth]{figs/null/chain0.pdf} -%\caption{This figure shows the evolution of one walker's parameter values for -%one element of the parameter vector $\vec{\theta}$ as a function of iteration -%number, demonstrating the completion of the burn-in phase.} -%\label{fig:chains} -%\end{figure} - -The autocorrelation time is effectively a measure of the efficiency of the method and can be described as the expected number of iterations necessary to accept a new sample independent of the current accepted sample. -A sampler that converges faster will have a smaller autocorrelation time, and smaller autocorrelation times are preferable because it means fewer iterations are wasted on non-independent samples when independent samples are desired. -See \citet{foreman-mackey_emcee_2013} for a more complete exploration of the autocorrelation time. -In all tests discussed here, autocorrelation times across walkers and parameters were approximately 20, meaning two samples 20 or more iterations apart were independent, a satisfactory level of efficiency. -Low autocorrelation times are a necessary but not always sufficient convergence condition, as the autocorrelation times calculated for tests in this paper were constant across all sub-runs, even those that were obviously burning in. - -The Gelman-Rubin statistic -\begin{equation} -\label{eqn:gr} -R_{k} = \sqrt{\frac{(1 - \frac{2}{I_{0}}) w_{k} + \frac{2}{I_{0}} b_{k}}{w_{k}}}, -\end{equation} -a weighted sum of the mean $w_{k}$ of the variances within individual walkers' chains and the variance $b_{k}$ between chains of different walkers $m$, is calculated over each sub-run $i$ to determine the duration of the burn-in period. -Convergence is achieved when the statistic approaches unity. - -\begin{acknowledgements} -AIM acknowledges support from the Max Planck Society and the Alexander von Humboldt Foundation in the framework of the Max Planck-Humboldt Research Award endowed by the Federal Ministry of Education and Research. -During the completion of this work, AIM was supported by National Science Foundation grant AST-1517237 and the U.S. Department of Energy, Office of Science, Office of Workforce Development for Teachers and Scientists, Office of Science Graduate Student Research (SCGSR) program, administered by the Oak Ridge Institute for Science and Education for the DOE under contract number DE‐SC0014664. -The authors thank Phil Marshall for advice on relevant examples, Elisabeth Krause for assistance with the \cosmolike code, Mohammadjavad Vakili for statistical insights, Geoffrey Ryan for programming advice, and Boris Leistedt for other helpful comments in the development of \Chippr. -\aim{TODO: Send draft around to Foreman-Mackey, Leistedt, others for feedback.} -This work was completed with generous nutritional support from the Center for Computational Astrophysics. -\end{acknowledgements} +%\section{Convergence Criteria} +%\label{app:acorr} +% +%\que{Cut convergence criteria section?} +% +%In addition to qualitative visual inspection of the chains, two quantities that probe the convergence of the sampler are used in this study, the autocorrelation time and the Gelman-Rubin convergence criterion. +%%\Fig{fig:chains} shows the %evolution of the values of one parameter of one walker over the course of all %iterations of the sampler. +% +%%\begin{figure} +%%%\includegraphics[width=0.5\textwidth]{figs/null/chain0.pdf} +%%\caption{This figure shows the evolution of one walker's parameter values for +%%one element of the parameter vector $\vec{\theta}$ as a function of iteration +%%number, demonstrating the completion of the burn-in phase.} +%%\label{fig:chains} +%%\end{figure} +% +%The autocorrelation time is effectively a measure of the efficiency of the method and can be described as the expected number of iterations necessary to accept a new sample independent of the current accepted sample. +%A sampler that converges faster will have a smaller autocorrelation time, and smaller autocorrelation times are preferable because it means fewer iterations are wasted on non-independent samples when independent samples are desired. +%See \citet{foreman-mackey_emcee_2013} for a more complete exploration of the autocorrelation time. +%In all tests discussed here, autocorrelation times across walkers and parameters were approximately 20, meaning two samples 20 or more iterations apart were independent, a satisfactory level of efficiency. +%Low autocorrelation times are a necessary but not always sufficient convergence condition, as the autocorrelation times calculated for tests in this paper were constant across all sub-runs, even those that were obviously burning in. +% +%The Gelman-Rubin statistic +%\begin{equation} +%\label{eqn:gr} +%R_{k} = \sqrt{\frac{(1 - \frac{2}{I_{0}}) w_{k} + \frac{2}{I_{0}} b_{k}}{w_{k}}}, +%\end{equation} +%a weighted sum of the mean $w_{k}$ of the variances within individual walkers' chains and the variance $b_{k}$ between chains of different walkers $m$, is calculated over each sub-run $i$ to determine the duration of the burn-in period. +%Convergence is achieved when the statistic approaches unity. \bibliographystyle{apj} \bibliography{draft} +%\aim{TODO: find way to cite Dance Your Ph.D. video} + \end{document} diff --git a/research/paper/figures/chippr/0single_lsst_lin_estimators.png b/research/paper/figures/chippr/0single_lsst_lin_estimators.png deleted file mode 100644 index 0aa81f6..0000000 Binary files a/research/paper/figures/chippr/0single_lsst_lin_estimators.png and /dev/null differ diff --git a/research/paper/figures/chippr/0single_lsst_mega_scatter.png b/research/paper/figures/chippr/0single_lsst_mega_scatter.png deleted file mode 100644 index 517b68a..0000000 Binary files a/research/paper/figures/chippr/0single_lsst_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/chippr/1single_lsst_lin_estimators.png b/research/paper/figures/chippr/1single_lsst_lin_estimators.png deleted file mode 100644 index 27f4b47..0000000 Binary files a/research/paper/figures/chippr/1single_lsst_lin_estimators.png and /dev/null differ diff --git a/research/paper/figures/chippr/1single_lsst_mega_scatter.png b/research/paper/figures/chippr/1single_lsst_mega_scatter.png deleted file mode 100644 index 3feb14d..0000000 Binary files a/research/paper/figures/chippr/1single_lsst_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/chippr/2single_lsst_lin_estimators.png b/research/paper/figures/chippr/2single_lsst_lin_estimators.png deleted file mode 100644 index 252e93c..0000000 Binary files a/research/paper/figures/chippr/2single_lsst_lin_estimators.png and /dev/null differ diff --git a/research/paper/figures/chippr/2single_lsst_mega_scatter.png b/research/paper/figures/chippr/2single_lsst_mega_scatter.png deleted file mode 100644 index 71ddc4c..0000000 Binary files a/research/paper/figures/chippr/2single_lsst_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/chippr/3single_lsst_lin_estimators.png b/research/paper/figures/chippr/3single_lsst_lin_estimators.png deleted file mode 100644 index 93ba5d5..0000000 Binary files a/research/paper/figures/chippr/3single_lsst_lin_estimators.png and /dev/null differ diff --git a/research/paper/figures/chippr/3single_lsst_mega_scatter.png b/research/paper/figures/chippr/3single_lsst_mega_scatter.png deleted file mode 100644 index 0dcc045..0000000 Binary files a/research/paper/figures/chippr/3single_lsst_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/chippr/cosmolike_inputs.png b/research/paper/figures/chippr/cosmolike_inputs.png deleted file mode 100644 index 3152bfe..0000000 Binary files a/research/paper/figures/chippr/cosmolike_inputs.png and /dev/null differ diff --git a/research/paper/figures/chippr/misspecified_log_estimators.png b/research/paper/figures/chippr/misspecified_log_estimators.png deleted file mode 100644 index 19c28eb..0000000 Binary files a/research/paper/figures/chippr/misspecified_log_estimators.png and /dev/null differ diff --git a/research/paper/figures/chippr/misspecified_mega_scatter.png b/research/paper/figures/chippr/misspecified_mega_scatter.png deleted file mode 100644 index 0affe45..0000000 Binary files a/research/paper/figures/chippr/misspecified_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/chippr/results_scatter1.png b/research/paper/figures/chippr/results_scatter1.png deleted file mode 100644 index 2763aff..0000000 Binary files a/research/paper/figures/chippr/results_scatter1.png and /dev/null differ diff --git a/research/paper/figures/chippr/results_scatter2.png b/research/paper/figures/chippr/results_scatter2.png deleted file mode 100644 index 01532d8..0000000 Binary files a/research/paper/figures/chippr/results_scatter2.png and /dev/null differ diff --git a/research/paper/figures/chippr/results_tmpr.png b/research/paper/figures/chippr/results_tmpr.png deleted file mode 100644 index a11585c..0000000 Binary files a/research/paper/figures/chippr/results_tmpr.png and /dev/null differ diff --git a/research/paper/figures/chippr/results_trpr.png b/research/paper/figures/chippr/results_trpr.png deleted file mode 100644 index a4fba18..0000000 Binary files a/research/paper/figures/chippr/results_trpr.png and /dev/null differ diff --git a/research/paper/figures/chippr/samplepzs_scatter1.png b/research/paper/figures/chippr/samplepzs_scatter1.png deleted file mode 100644 index 76e6048..0000000 Binary files a/research/paper/figures/chippr/samplepzs_scatter1.png and /dev/null differ diff --git a/research/paper/figures/chippr/samplepzs_scatter2.png b/research/paper/figures/chippr/samplepzs_scatter2.png deleted file mode 100644 index 15aab41..0000000 Binary files a/research/paper/figures/chippr/samplepzs_scatter2.png and /dev/null differ diff --git a/research/paper/figures/chippr/samplepzs_tmpr.png b/research/paper/figures/chippr/samplepzs_tmpr.png deleted file mode 100644 index 4ff85f7..0000000 Binary files a/research/paper/figures/chippr/samplepzs_tmpr.png and /dev/null differ diff --git a/research/paper/figures/chippr/samplepzs_trpr.png b/research/paper/figures/chippr/samplepzs_trpr.png deleted file mode 100644 index f7529b3..0000000 Binary files a/research/paper/figures/chippr/samplepzs_trpr.png and /dev/null differ diff --git a/research/paper/figures/chippr/single_lsst-mega_scatter.png b/research/paper/figures/chippr/single_lsst-mega_scatter.png new file mode 100644 index 0000000..32ea9f2 Binary files /dev/null and b/research/paper/figures/chippr/single_lsst-mega_scatter.png differ diff --git a/research/paper/figures/chippr/single_lsst_tmpr-mega_scatter.png b/research/paper/figures/chippr/single_lsst_tmpr-mega_scatter.png new file mode 100644 index 0000000..601499d Binary files /dev/null and b/research/paper/figures/chippr/single_lsst_tmpr-mega_scatter.png differ diff --git a/research/paper/figures/chippr/single_lsst_tmpr_log_estimators.png b/research/paper/figures/chippr/single_lsst_tmpr_log_estimators.png new file mode 100644 index 0000000..ddc3674 Binary files /dev/null and b/research/paper/figures/chippr/single_lsst_tmpr_log_estimators.png differ diff --git a/research/paper/figures/chippr/single_lsst_tmpr_wrong_log_estimators.png b/research/paper/figures/chippr/single_lsst_tmpr_wrong_log_estimators.png new file mode 100644 index 0000000..7a0f28f Binary files /dev/null and b/research/paper/figures/chippr/single_lsst_tmpr_wrong_log_estimators.png differ diff --git a/research/paper/figures/chippr/single_lsst_trpr-mega_scatter.png b/research/paper/figures/chippr/single_lsst_trpr-mega_scatter.png new file mode 100644 index 0000000..9d14e8d Binary files /dev/null and b/research/paper/figures/chippr/single_lsst_trpr-mega_scatter.png differ diff --git a/research/paper/figures/chippr/single_lsst_trpr_log_estimators.png b/research/paper/figures/chippr/single_lsst_trpr_log_estimators.png new file mode 100644 index 0000000..32c0abf Binary files /dev/null and b/research/paper/figures/chippr/single_lsst_trpr_log_estimators.png differ diff --git a/research/paper/figures/chippr/single_lsst_trpr_wrong_log_estimators.png b/research/paper/figures/chippr/single_lsst_trpr_wrong_log_estimators.png new file mode 100644 index 0000000..b6d849e Binary files /dev/null and b/research/paper/figures/chippr/single_lsst_trpr_wrong_log_estimators.png differ diff --git a/research/paper/figures/chippr/single_uout-mega_scatter.png b/research/paper/figures/chippr/single_uout-mega_scatter.png index e18f76f..7d6e67f 100644 Binary files a/research/paper/figures/chippr/single_uout-mega_scatter.png and b/research/paper/figures/chippr/single_uout-mega_scatter.png differ diff --git a/research/paper/figures/chippr/single_uout_log_estimators.png b/research/paper/figures/chippr/single_uout_log_estimators.png index fc8972c..f1ec401 100644 Binary files a/research/paper/figures/chippr/single_uout_log_estimators.png and b/research/paper/figures/chippr/single_uout_log_estimators.png differ diff --git a/research/paper/figures/chippr/single_uout_mega_scatter.png b/research/paper/figures/chippr/single_uout_mega_scatter.png deleted file mode 100644 index d0ac1aa..0000000 Binary files a/research/paper/figures/chippr/single_uout_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/chippr/single_varsigmas-mega_scatter.png b/research/paper/figures/chippr/single_varsigmas-mega_scatter.png new file mode 100644 index 0000000..d1c3ea4 Binary files /dev/null and b/research/paper/figures/chippr/single_varsigmas-mega_scatter.png differ diff --git a/research/paper/figures/chippr/single_varsigmas_log_estimators.png b/research/paper/figures/chippr/single_varsigmas_log_estimators.png new file mode 100644 index 0000000..767f517 Binary files /dev/null and b/research/paper/figures/chippr/single_varsigmas_log_estimators.png differ diff --git a/research/paper/figures/chippr/thesis_eout-mega_scatter.png b/research/paper/figures/chippr/thesis_eout-mega_scatter.png new file mode 100644 index 0000000..cc617a4 Binary files /dev/null and b/research/paper/figures/chippr/thesis_eout-mega_scatter.png differ diff --git a/research/paper/figures/chippr/thesis_eout_log_estimators.png b/research/paper/figures/chippr/thesis_eout_log_estimators.png index aa1012e..e109bef 100644 Binary files a/research/paper/figures/chippr/thesis_eout_log_estimators.png and b/research/paper/figures/chippr/thesis_eout_log_estimators.png differ diff --git a/research/paper/figures/chippr/thesis_eout_mega_scatter.png b/research/paper/figures/chippr/thesis_eout_mega_scatter.png deleted file mode 100644 index 2bf9602..0000000 Binary files a/research/paper/figures/chippr/thesis_eout_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/chippr/thesis_hivarsig-mega_scatter.png b/research/paper/figures/chippr/thesis_hivarsig-mega_scatter.png index a3d1963..f8bd7b7 100644 Binary files a/research/paper/figures/chippr/thesis_hivarsig-mega_scatter.png and b/research/paper/figures/chippr/thesis_hivarsig-mega_scatter.png differ diff --git a/research/paper/figures/chippr/thesis_hivarsig_log_estimators.png b/research/paper/figures/chippr/thesis_hivarsig_log_estimators.png new file mode 100644 index 0000000..c2dafdc Binary files /dev/null and b/research/paper/figures/chippr/thesis_hivarsig_log_estimators.png differ diff --git a/research/paper/figures/chippr/thesis_neghivarbias-mega_scatter.png b/research/paper/figures/chippr/thesis_neghivarbias-mega_scatter.png index f75b0a2..fde6c08 100644 Binary files a/research/paper/figures/chippr/thesis_neghivarbias-mega_scatter.png and b/research/paper/figures/chippr/thesis_neghivarbias-mega_scatter.png differ diff --git a/research/paper/figures/chippr/thesis_neghivarbias_log_estimators.png b/research/paper/figures/chippr/thesis_neghivarbias_log_estimators.png index 7940928..4fa649d 100644 Binary files a/research/paper/figures/chippr/thesis_neghivarbias_log_estimators.png and b/research/paper/figures/chippr/thesis_neghivarbias_log_estimators.png differ diff --git a/research/paper/figures/chippr/thesis_neghivarbias_mega_scatter.png b/research/paper/figures/chippr/thesis_neghivarbias_mega_scatter.png deleted file mode 100644 index 7cf2678..0000000 Binary files a/research/paper/figures/chippr/thesis_neghivarbias_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/chippr/thesis_rout-mega_scatter.png b/research/paper/figures/chippr/thesis_rout-mega_scatter.png new file mode 100644 index 0000000..f16cbf7 Binary files /dev/null and b/research/paper/figures/chippr/thesis_rout-mega_scatter.png differ diff --git a/research/paper/figures/chippr/thesis_rout_log_estimators.png b/research/paper/figures/chippr/thesis_rout_log_estimators.png index 3e7ba5c..b9d770d 100644 Binary files a/research/paper/figures/chippr/thesis_rout_log_estimators.png and b/research/paper/figures/chippr/thesis_rout_log_estimators.png differ diff --git a/research/paper/figures/chippr/thesis_rout_mega_scatter.png b/research/paper/figures/chippr/thesis_rout_mega_scatter.png deleted file mode 100644 index 1edce56..0000000 Binary files a/research/paper/figures/chippr/thesis_rout_mega_scatter.png and /dev/null differ diff --git a/research/paper/figures/jain05.png b/research/paper/figures/jain05.png index d808a1f..fa660ca 100644 Binary files a/research/paper/figures/jain05.png and b/research/paper/figures/jain05.png differ diff --git a/research/scripts/multi_replot_script.py b/research/scripts/multi_replot_script.py index a521a10..6eec47a 100644 --- a/research/scripts/multi_replot_script.py +++ b/research/scripts/multi_replot_script.py @@ -111,6 +111,6 @@ def just_plot(given_key): test_info['name'] = test_name all_tests[test_name] = test_info - nps = mp.cpu_count() + nps = mp.cpu_count() - 1 pool = mp.Pool(nps) pool.map(just_plot, all_tests.keys())