From 4ac0c74e643033fe67a2fa39ba5c65c79479b7e3 Mon Sep 17 00:00:00 2001 From: delucchi-cmu Date: Wed, 21 Jun 2023 12:22:46 -0400 Subject: [PATCH 1/6] Checkpoint --- docs/catalogs/allwise.rst | 6 +- docs/catalogs/neowise.rst | 6 +- docs/catalogs/tic.rst | 6 +- docs/guide/association.rst | 9 + docs/guide/catalog_arguments.rst | 20 ++ docs/guide/{resume.rst => catalog_resume.rst} | 0 docs/guide/command_line.rst | 7 - docs/guide/contributing.rst | 17 +- docs/guide/index_table.rst | 8 + docs/guide/margin_cache.rst | 9 + docs/guide/overview.rst | 36 ++- docs/index.rst | 8 +- docs/notebooks.rst | 2 +- docs/notebooks/estimate_pixel_threshold.ipynb | 206 ++++++++++++++++++ docs/notebooks/intro_notebook.ipynb | 111 ---------- src/hipscat_import/catalog/arguments.py | 11 +- 16 files changed, 321 insertions(+), 141 deletions(-) create mode 100644 docs/guide/association.rst create mode 100644 docs/guide/catalog_arguments.rst rename docs/guide/{resume.rst => catalog_resume.rst} (100%) delete mode 100644 docs/guide/command_line.rst create mode 100644 docs/guide/index_table.rst create mode 100644 docs/guide/margin_cache.rst create mode 100755 docs/notebooks/estimate_pixel_threshold.ipynb delete mode 100644 docs/notebooks/intro_notebook.ipynb diff --git a/docs/catalogs/allwise.rst b/docs/catalogs/allwise.rst index 0a229ca9..7e149939 100644 --- a/docs/catalogs/allwise.rst +++ b/docs/catalogs/allwise.rst @@ -27,9 +27,9 @@ Example import import pandas as pd - import hipscat_import.run_import as runner - from hipscat_import.arguments import ImportArguments - from hipscat_import.file_readers import CsvReader + import hipscat_import.catalog.run_import as runner + from hipscat_import.catalog.arguments import ImportArguments + from hipscat_import.catalog.hipscat_import.file_readers import CsvReader # Load the column names and types from a side file. type_frame = pd.read_csv("allwise_types.csv") diff --git a/docs/catalogs/neowise.rst b/docs/catalogs/neowise.rst index ea61c9b5..e8f30891 100644 --- a/docs/catalogs/neowise.rst +++ b/docs/catalogs/neowise.rst @@ -27,9 +27,9 @@ Example import import pandas as pd - import hipscat_import.run_import as runner - from hipscat_import.arguments import ImportArguments - from hipscat_import.file_readers import CsvReader + import hipscat_import.catalog.run_import as runner + from hipscat_import.catalog.arguments import ImportArguments + from hipscat_import.catalog.hipscat_import.file_readers import CsvReader # Load the column names and types from a side file. type_frame = pd.read_csv("neowise_types.csv") diff --git a/docs/catalogs/tic.rst b/docs/catalogs/tic.rst index e044b1ee..dc7aa908 100644 --- a/docs/catalogs/tic.rst +++ b/docs/catalogs/tic.rst @@ -27,9 +27,9 @@ Example import import pandas as pd - import hipscat_import.run_import as runner - from hipscat_import.arguments import ImportArguments - from hipscat_import.file_readers import CsvReader + import hipscat_import.catalog.run_import as runner + from hipscat_import.catalog.arguments import ImportArguments + from hipscat_import.catalog.hipscat_import.file_readers import CsvReader type_frame = pd.read_csv("tic_types.csv") type_map = dict(zip(type_frame["name"], type_frame["type"])) diff --git a/docs/guide/association.rst b/docs/guide/association.rst new file mode 100644 index 00000000..57537648 --- /dev/null +++ b/docs/guide/association.rst @@ -0,0 +1,9 @@ +Association Table +=============================================================================== + +See the API documentation for :py:class:`hipscat_import.association.arguments.AssociationArguments` + +.. note:: + + TODO - write the rest + diff --git a/docs/guide/catalog_arguments.rst b/docs/guide/catalog_arguments.rst new file mode 100644 index 00000000..d6f02004 --- /dev/null +++ b/docs/guide/catalog_arguments.rst @@ -0,0 +1,20 @@ +Catalog Import Arguments +=============================================================================== + +This page discusses a few topics around setting up a catalog pipeline. + +For a full list of the available arguments, see the API documentation for +:py:class:`hipscat_import.catalog.arguments.ImportArguments` + +Reading input files +------------------------------------------------------------------------------- + +Catalog import reads through a list of files and converts them into a hipscatted catalog. + +There are a few ways to specify the files to read: + + +Other Topics +------------------------------------------------------------------------------- + +* :doc:`catalog_resume` \ No newline at end of file diff --git a/docs/guide/resume.rst b/docs/guide/catalog_resume.rst similarity index 100% rename from docs/guide/resume.rst rename to docs/guide/catalog_resume.rst diff --git a/docs/guide/command_line.rst b/docs/guide/command_line.rst deleted file mode 100644 index 34c36c40..00000000 --- a/docs/guide/command_line.rst +++ /dev/null @@ -1,7 +0,0 @@ -Command Line Arguments -=============================================================================== - -TODO - -Arguments -------------------------------------------------------------------------------- \ No newline at end of file diff --git a/docs/guide/contributing.rst b/docs/guide/contributing.rst index 9faf8a9b..bacb1a91 100644 --- a/docs/guide/contributing.rst +++ b/docs/guide/contributing.rst @@ -32,9 +32,22 @@ Most folks use conda for virtual environments. You may want to as well. $ pip install -e . .. tip:: - Installing dev dependencies on Mac + Installing on Mac - (Make sure to include the single quotes) + ``healpy`` is a very necessary dependency for hipscat libraries at this time, but + native prebuilt binaries for healpy on Apple Silicon Macs + `do not yet exist `_, + so it's recommended to install via conda before proceeding to hipscat-import. + + .. code-block:: bash + + $ conda config --add channels conda-forge + $ conda install healpy + $ git clone https://github.com/astronomy-commons/hipscat-import + $ cd hipscat-import + $ pip install -e . + + When installing dev dependencies, make sure to include the single quotes. .. code-block:: bash diff --git a/docs/guide/index_table.rst b/docs/guide/index_table.rst new file mode 100644 index 00000000..a5f6d1bb --- /dev/null +++ b/docs/guide/index_table.rst @@ -0,0 +1,8 @@ +Index Table +=============================================================================== + +See the API documentation for :py:class:`hipscat_import.index.arguments.IndexArguments` + +.. note:: + + TODO - write the rest \ No newline at end of file diff --git a/docs/guide/margin_cache.rst b/docs/guide/margin_cache.rst new file mode 100644 index 00000000..1c97dcb6 --- /dev/null +++ b/docs/guide/margin_cache.rst @@ -0,0 +1,9 @@ +Margin Cache +=============================================================================== + +See the API documentation for +:py:class:`hipscat_import.margin_cache.margin_cache_arguments.MarginCacheArguments` + +.. note:: + + TODO - write the rest \ No newline at end of file diff --git a/docs/guide/overview.rst b/docs/guide/overview.rst index 1b9bdc6d..63e4f9e4 100644 --- a/docs/guide/overview.rst +++ b/docs/guide/overview.rst @@ -8,8 +8,38 @@ Installation pip install hipscat-import -Other Topics +.. tip:: + Installing on Mac + + ``healpy`` is a very necessary dependency for hipscat libraries at this time, but + native prebuilt binaries for healpy on Apple Silicon Macs + `do not yet exist `_, + so it's recommended to install via conda before proceeding to hipscat-import. + + .. code-block:: bash + + $ conda config --add channels conda-forge + $ conda install healpy + +Setting up a pipeline ------------------------------------------------------------------------------- -* :doc:`command_line` -* :doc:`resume` +For each type of dataset the hipscat-import tool can generate, there is an argument +container class that you will need to instantiate and populate with relevant arguments. + +See dataset-specific notes on arguments: + +* :doc:`catalog_arguments` (most common) +* :doc:`margin_cache` +* :doc:`association` +* :doc:`index_table` + +Once you have created your arguments object, you pass it into the pipeline control, +and then wait: + +.. code-block:: python + + import hipscat_import.control as runner + + args = ... + runner.run(args) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index d0ecaaaf..d849780b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,11 +5,13 @@ Utility for ingesting large survey data into HiPSCat structure. .. toctree:: :maxdepth: 1 - :caption: Importing Catalogs + :caption: Importing Datasets guide/overview - guide/command_line - guide/resume + guide/catalog_arguments + guide/margin_cache + guide/association + guide/index_table Notebooks .. toctree:: diff --git a/docs/notebooks.rst b/docs/notebooks.rst index 7f7e544d..5e218f09 100644 --- a/docs/notebooks.rst +++ b/docs/notebooks.rst @@ -3,4 +3,4 @@ Notebooks .. toctree:: - Introducing Jupyter Notebooks + Estimate Pixel Threshold diff --git a/docs/notebooks/estimate_pixel_threshold.ipynb b/docs/notebooks/estimate_pixel_threshold.ipynb new file mode 100755 index 00000000..e0adc254 --- /dev/null +++ b/docs/notebooks/estimate_pixel_threshold.ipynb @@ -0,0 +1,206 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "98af180d", + "metadata": {}, + "source": [ + "# Estimate pixel threshold\n", + "\n", + "For best performance, we try to keep catalog parquet files between 200-800MB in size.\n", + "\n", + "**Background**\n", + "\n", + "When creating a new catalog through the hipscat-import process, we try to create partitions with approximately the same number of rows per partition. This isn't perfect, because the sky is uneven, but we still try to create smaller-area pixels in more dense areas, and larger-area pixels in less dense areas. We use the argument `pixel_threshold` and will split a partition into smaller healpix pixels until the number of rows is smaller than `pixel_threshold`.\n", + "\n", + "We do this to increase parallelization of reads and downstream analysis: if the files are around the same size, and operations on each partition take around the same amount of time, we're not as likely to be waiting on a single process to complete for the whole pipeline to complete.\n", + "\n", + "In addition, a single catalog file should not exceed a couple GB - we're going to need to read the whole thing into memory, so it needs to fit!\n", + "\n", + "**Objective**\n", + "\n", + "In this notebook, we'll go over *one* strategy for estimating the `pixel_threshold` argument you can use when importing a new catalog into hipscat format.\n", + "\n", + "This is not guaranteed to give you optimal results, but it could give you some hints toward *better* results." + ] + }, + { + "cell_type": "markdown", + "id": "eb86458c", + "metadata": {}, + "source": [ + "## Create a sample parquet file\n", + "\n", + "The first step is to read in your survey data in its original form, and convert a sample into parquet. This has a few benefits:\n", + "- parquet uses compression in various ways, and by creating the sample, we can get a sense of both the overall and field-level compression with real dat\n", + "- using the importer `FileReader` interface now sets you up for more success when you get around to importing!\n", + "\n", + "If your data is already in parquet format, just change the `sample_parquet_file` path to an existing file, and don't run the second cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5dd94480", + "metadata": {}, + "outputs": [], + "source": [ + "### Change this path!!!\n", + "sample_parquet_file=\"../_build/sample.parquet\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6a53db0", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from hipscat_import.catalog.file_readers import CsvReader\n", + "\n", + "### Change this path!!!\n", + "input_file=\"../../tests/hipscat_import/data/small_sky/catalog.csv\"\n", + "\n", + "file_reader = CsvReader(\n", + " chunksize=5_000\n", + " )\n", + "\n", + "next(file_reader.read(input_file)).to_parquet(sample_parquet_file)" + ] + }, + { + "cell_type": "markdown", + "id": "124eb444", + "metadata": {}, + "source": [ + "## Inspect parquet file and metadata\n", + "\n", + "Now that we have parsed our survey data into parquet, we can check what the data will look like when it's imported into hipscat format.\n", + "\n", + "If you're just here to get a naive estimate for your pixel threshold, we'll do that first, then take a look at some other parquet characteristics later for the curious." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9f0e279", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pyarrow.parquet as pq\n", + "\n", + "sample_file_size = os.path.getsize(sample_parquet_file)\n", + "parquet_file = pq.ParquetFile(sample_parquet_file)\n", + "num_rows = parquet_file.metadata.num_rows\n", + "\n", + "## 100MB\n", + "ideal_file_small = 100 *1024*1024\n", + "## 800MB\n", + "ideal_file_large = 800 *1024*1024\n", + "\n", + "threshold_small = ideal_file_small/sample_file_size*num_rows\n", + "threshold_large = ideal_file_large/sample_file_size*num_rows\n", + "\n", + "print(f\"threshold between {int(threshold_small):_} and {int(threshold_large):_}\")" + ] + }, + { + "cell_type": "markdown", + "id": "23971c38", + "metadata": {}, + "source": [ + "## Want to see more?\n", + "\n", + "I'm so glad you're still here! I have more to show you!\n", + "\n", + "The first step below shows us the file-level metadata, as stored by parquet. The number of columns here SHOULD match your expectations on the number of columns in your survey data.\n", + "\n", + "The `serialized_size` value is just the size of the total metadata, not the size of the file. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc402acf", + "metadata": {}, + "outputs": [], + "source": [ + "import pyarrow.parquet as pq\n", + "\n", + "parquet_file = pq.ParquetFile(sample_parquet_file)\n", + "print(parquet_file.metadata)" + ] + }, + { + "cell_type": "markdown", + "id": "7835b6d9", + "metadata": {}, + "source": [ + "The next step is to look at the column-level metadata. You can check that the on-disk type of each column matches your expectation of the data. Note that for some integer types, the on-disk type may be a smaller int than originally set (e.g. `bitWidth=8` or `16`). This is part of parquet's multi-part compression strategy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ff8fb4d", + "metadata": {}, + "outputs": [], + "source": [ + "print(parquet_file.schema)" + ] + }, + { + "cell_type": "markdown", + "id": "5c2b593b", + "metadata": {}, + "source": [ + "Parquet will also perform some column-level compression, so not all columns with the same type will take up the same space on disk.\n", + "\n", + "Below, we inspect the row and column group metadata to show the compressed size of the fields on disk. The last column, `percent`, show the percent of total size taken up by the column.\n", + "\n", + "You *can* use this to inform which columns you keep when importing a catalog into hipscat format. e.g. if some columns are less useful for your science, and take up a lot of space, maybe leave them out!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dcf152f2", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "num_cols = parquet_file.metadata.num_columns\n", + "num_row_groups = parquet_file.metadata.num_row_groups\n", + "sizes = np.zeros(num_cols)\n", + "\n", + "for rg in range(num_row_groups):\n", + " for col in range (num_cols):\n", + " sizes[col] += parquet_file.metadata.row_group(rg).column(col).total_compressed_size\n", + "\n", + "## This is just an attempt at pretty formatting\n", + "percents = [f\"{s/sizes.sum()*100:.1f}\" for s in sizes]\n", + "pd.DataFrame({\"name\":parquet_file.schema.names, \"size\":sizes.astype(int), \"percent\": percents}).sort_values(\"size\", ascending=False)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/docs/notebooks/intro_notebook.ipynb b/docs/notebooks/intro_notebook.ipynb deleted file mode 100644 index 2e7779f5..00000000 --- a/docs/notebooks/intro_notebook.ipynb +++ /dev/null @@ -1,111 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "accepting-editor", - "metadata": { - "cell_marker": "\"\"\"" - }, - "source": [ - "# Introducing Jupyter Notebooks\n", - "\n", - "_(The example used here is JamesALeedham's notebook: [intro.ipynb](https://github.com/JamesALeedham/Sphinx-Autosummary-Recursion/blob/master/docs/notebooks/intro.ipynb))_\n", - "\n", - "First, set up the environment:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "actual-thirty", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib\n", - "import matplotlib.pyplot as pl\n", - "import numpy as np\n", - "\n", - "try:\n", - " from IPython import get_ipython\n", - " get_ipython().run_line_magic('matplotlib', 'inline')\n", - "except AttributeError:\n", - " print('Magic function can only be used in IPython environment')\n", - " matplotlib.use('Agg')\n", - "\n", - "pl.rcParams[\"figure.figsize\"] = [15, 8]" - ] - }, - { - "cell_type": "markdown", - "id": "coral-upper", - "metadata": { - "cell_marker": "\"\"\"", - "lines_to_next_cell": 1 - }, - "source": [ - "Then, define a function that creates a pretty graph:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "funded-protection", - "metadata": { - "lines_to_next_cell": 1 - }, - "outputs": [], - "source": [ - "def SineAndCosineWaves():\n", - " # Get a large number of X values for a nice smooth curve. Using Pi as np.sin requires radians...\n", - " x = np.linspace(0, 2 * np.pi, 180)\n", - " # Convert radians to degrees to make for a meaningful X axis (1 radian = 57.29* degrees)\n", - " xdeg = 57.29577951308232 * np.array(x)\n", - " # Calculate the sine of each value of X\n", - " y = np.sin(x)\n", - " # Calculate the cosine of each value of X\n", - " z = np.cos(x)\n", - " # Plot the sine wave in blue, using degrees rather than radians on the X axis\n", - " pl.plot(xdeg, y, color='blue', label='Sine wave')\n", - " # Plot the cos wave in green, using degrees rather than radians on the X axis\n", - " pl.plot(xdeg, z, color='green', label='Cosine wave')\n", - " pl.xlabel(\"Degrees\")\n", - " # More sensible X axis values\n", - " pl.xticks(np.arange(0, 361, 45))\n", - " pl.legend()\n", - " pl.show()" - ] - }, - { - "cell_type": "markdown", - "id": "thorough-cutting", - "metadata": { - "cell_marker": "\"\"\"" - }, - "source": [ - "Finally, call that function to display the graph:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "imported-uruguay", - "metadata": {}, - "outputs": [], - "source": [ - "SineAndCosineWaves()" - ] - } - ], - "metadata": { - "jupytext": { - "cell_markers": "\"\"\"" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/src/hipscat_import/catalog/arguments.py b/src/hipscat_import/catalog/arguments.py index 7fa98135..23bff4b4 100644 --- a/src/hipscat_import/catalog/arguments.py +++ b/src/hipscat_import/catalog/arguments.py @@ -60,11 +60,11 @@ class ImportArguments(RuntimeArguments): pixels that don't meed the threshold""" pixel_threshold: int = 1_000_000 """maximum number of rows for a single resulting pixel. - we may combine hierarchically until we near the `pixel_threshold`""" + we may combine hierarchically until we near the ``pixel_threshold``""" mapping_healpix_order: int = -1 """healpix order to use when mapping. will be - `highest_healpix_order` unless a positive value is provided for - `constant_healpix_order`""" + ``highest_healpix_order`` unless a positive value is provided for + ``constant_healpix_order``""" debug_stats_only: bool = False """do not perform a map reduce and don't create a new catalog. generate the partition info""" @@ -178,8 +178,9 @@ def additional_runtime_provenance_info(self): def check_healpix_order_range( order, field_name, lower_bound=0, upper_bound=hipscat_id.HIPSCAT_ID_HEALPIX_ORDER ): - """Helper method to heck if the `order` is within the range determined by the - `lower_bound` and `upper_bound`, inclusive. + """Helper method to heck if the ``order`` is within the range determined by the + ``lower_bound`` and ``upper_bound``, inclusive. + Args: order (int): healpix order to check field_name (str): field name to use in the error message From b315284c0890be0dec65cc8b69be1524ae31c05f Mon Sep 17 00:00:00 2001 From: delucchi-cmu Date: Wed, 21 Jun 2023 15:40:12 -0400 Subject: [PATCH 2/6] Checkpoint --- docs/catalogs/advanced.rst | 37 ++++++ docs/catalogs/arguments.rst | 115 ++++++++++++++++++ docs/catalogs/debug.rst | 16 +++ docs/catalogs/overview.rst | 13 -- docs/catalogs/{ => public}/allwise.rst | 16 +-- docs/catalogs/public/index.rst | 21 ++++ docs/catalogs/{ => public}/neowise.rst | 6 +- docs/catalogs/{ => public}/tic.rst | 6 +- docs/catalogs/public/zubercal.rst | 85 +++++++++++++ .../resume.rst} | 0 docs/guide/catalog_arguments.rst | 20 --- docs/guide/contact.rst | 11 ++ docs/guide/contributing.rst | 5 + docs/guide/overview.rst | 45 ------- docs/index.rst | 73 +++++++++-- 15 files changed, 365 insertions(+), 104 deletions(-) create mode 100644 docs/catalogs/advanced.rst create mode 100644 docs/catalogs/arguments.rst create mode 100644 docs/catalogs/debug.rst delete mode 100644 docs/catalogs/overview.rst rename docs/catalogs/{ => public}/allwise.rst (83%) create mode 100644 docs/catalogs/public/index.rst rename docs/catalogs/{ => public}/neowise.rst (91%) rename docs/catalogs/{ => public}/tic.rst (91%) create mode 100644 docs/catalogs/public/zubercal.rst rename docs/{guide/catalog_resume.rst => catalogs/resume.rst} (100%) delete mode 100644 docs/guide/catalog_arguments.rst create mode 100644 docs/guide/contact.rst delete mode 100644 docs/guide/overview.rst diff --git a/docs/catalogs/advanced.rst b/docs/catalogs/advanced.rst new file mode 100644 index 00000000..e4249418 --- /dev/null +++ b/docs/catalogs/advanced.rst @@ -0,0 +1,37 @@ +Advanced Usage +=============================================================================== + +We aim to support ingestion of a lot of kinds of catalog data. Here, we discuss +some ways you can tune the import pipeline for different kinds of data. + +.. tip:: + Reach out! + + If you have some *very* interesting data that isn't well-supported by this + pipeline, we want to hear about it! :doc:`/guide/contact` + + +``add_hipscat_index`` +------------------------------------------------------------------------------- + +TODO + +``use_schema_file`` +------------------------------------------------------------------------------- + +TODO + +``debug_stats_only`` +------------------------------------------------------------------------------- + +TODO + +``epoch`` +------------------------------------------------------------------------------- + +TODO + +``catalog_type`` +------------------------------------------------------------------------------- + +TODO diff --git a/docs/catalogs/arguments.rst b/docs/catalogs/arguments.rst new file mode 100644 index 00000000..c2dcac62 --- /dev/null +++ b/docs/catalogs/arguments.rst @@ -0,0 +1,115 @@ +Catalog Import Arguments +=============================================================================== + +This page discusses a few topics around setting up a catalog pipeline. + +For a full list of the available arguments, see the API documentation for +:py:class:`hipscat_import.catalog.arguments.ImportArguments` + +Reading input files +------------------------------------------------------------------------------- + +Catalog import reads through a list of files and converts them into a hipscatted catalog. + +Which files? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are a few ways to specify the files to read: + +* ``input_path`` + ``input_format``: + will search for files ending with the format string in the indicated directory. +* ``input_file_list``: + a list of fully-specified paths you want to read. + + * this strategy can be useful to first run the import on a single input + file and validate the input, then run again on the full input set, or + to debug a single input file with odd behavior. + * if you have a mix of files in your target directory, you can use a glob + statement like the following to gather input files: + +.. code-block:: python + + in_file_paths = glob.glob("/data/object_and_source/object**.csv") + in_file_paths.sort() + +How to read them? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Specify an instance of ``InputReader`` for the ``file_reader`` parameter. + +see the API documentation for +:py:class:`hipscat_import.catalog.file_readers.InputReader` + +We use the ``InputReader`` class to read files in chunks and pass the chunks +along to the map/reduce stages. We've provided reference implementations for +reading CSV, FITS, and Parquet input files, but you can subclass the reader +type to suit whatever input files you've got. + +.. code-block:: python + + class StarrReader(InputReader): + """Class for fictional Starr file format.""" + def __init__(self, chunksize=500_000, **kwargs): + self.chunksize = chunksize + self.kwargs = kwargs + + def read(self, input_file): + starr_file = starr_io.read_table(input_file, **self.kwargs) + for smaller_table in starr_file.to_batches(max_chunksize=self.chunksize): + smaller_table = filter_nonsense(smaller_table) + yield smaller_table.to_pandas() + + def provenance_info(self) -> dict: + provenance_info = { + "input_reader_type": "StarrReader", + "chunksize": self.chunksize, + } + return provenance_info + + ... + + args = ImportArguments( + ... + ## Locates files like "/directory/to/files/**starr" + input_path="/directory/to/files/", + input_format="starr", + ## NB - you need the parens here! + file_reader=StarrReader(), + + ) + +Which fields? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Specify the ``ra_column`` and ``dec_column`` for the dataset. + +There are two fields that we require in order to make a valid hipscatted +catalog, the right ascension and declination. At this time, this is the only +supported system for celestial coordinates. + + +Healpix order and thresholds +------------------------------------------------------------------------------- + +Details for ``pixel_threshold``, ``highest_healpix_order``, and +``constant_healpix_order`` arguments + +When creating a new catalog through the hipscat-import process, we try to +create partitions with approximately the same number of rows per partition. +This isn't perfect, because the sky is uneven, but we still try to create +smaller-area pixels in more dense areas, and larger-area pixels in less dense +areas. + +We use the argument ``pixel_threshold`` and will split a partition into +smaller healpix pixels until the number of rows is smaller than ``pixel_threshold``. +We will only split by healpix pixels up to the ``highest_healpix_order``. If we +would need to split further, we'll throw an error at the "Binning" stage, and you +should adjust your parameters. + +For more discussion of the ``pixel_threshold`` argument and a strategy for setting +this parameter, see notebook :doc:`/notebooks/estimate_pixel_threshold` + +Alternatively, you can use the ``constant_healpix_order`` argument. This will +**ignore** both of the ``pixel_threshold`` and ``highest_healpix_order`` arguments +and the catalog will be partitioned by healpix pixels at the +``constant_healpix_order``. This can be useful for very sparse datasets. \ No newline at end of file diff --git a/docs/catalogs/debug.rst b/docs/catalogs/debug.rst new file mode 100644 index 00000000..594bb934 --- /dev/null +++ b/docs/catalogs/debug.rst @@ -0,0 +1,16 @@ +Debugging Tips +======================================================================================== + +.. tip:: + If you're struggling with your dataset after looking over these tips, reach out! + + :doc:`/guide/contact` + +Reduce step +------------------------------------------------------------------------------- + +Errors like: + + ``` + Exception: "ArrowNotImplementedError('Unsupported cast from string to null using function cast_null')" + ``` \ No newline at end of file diff --git a/docs/catalogs/overview.rst b/docs/catalogs/overview.rst deleted file mode 100644 index b7073268..00000000 --- a/docs/catalogs/overview.rst +++ /dev/null @@ -1,13 +0,0 @@ -Overview -=============================================================================== - -The LINCC Frameworks team has built the import tool hoping to handle catalogs -in various formats. We've learned some lessons in importing public data sets, -and provide steps to import those catalogs in case they help anyone else. - -Catalogs -------------------------------------------------------------------------------- - -* :doc:`allwise` -* :doc:`neowise` -* :doc:`tic` diff --git a/docs/catalogs/allwise.rst b/docs/catalogs/public/allwise.rst similarity index 83% rename from docs/catalogs/allwise.rst rename to docs/catalogs/public/allwise.rst index 7e149939..a66e5039 100644 --- a/docs/catalogs/allwise.rst +++ b/docs/catalogs/public/allwise.rst @@ -27,16 +27,16 @@ Example import import pandas as pd - import hipscat_import.catalog.run_import as runner + import hipscat_import.pipeline as runner from hipscat_import.catalog.arguments import ImportArguments - from hipscat_import.catalog.hipscat_import.file_readers import CsvReader + from hipscat_import.catalog.file_readers import CsvReader # Load the column names and types from a side file. type_frame = pd.read_csv("allwise_types.csv") type_map = dict(zip(type_frame["name"], type_frame["type"])) args = ImportArguments( - catalog_name="allwise", + output_catalog_name="allwise", input_path="/path/to/allwise/", input_format="csv.bz2", file_reader=CsvReader( @@ -46,9 +46,11 @@ Example import type_map=type_map, chunksize=250_000, ).read, - ra_column="RA", - dec_column="DEC", - id_column="SOURCE_ID", + ra_column="ra", + dec_column="dec", + id_column="source_id", + pixel_threshold=1_000_000, + highest_healpix_order=7, output_path="/path/to/catalogs/", ) - runner.run(args) + runner.pipeline(args) diff --git a/docs/catalogs/public/index.rst b/docs/catalogs/public/index.rst new file mode 100644 index 00000000..389862a3 --- /dev/null +++ b/docs/catalogs/public/index.rst @@ -0,0 +1,21 @@ +Public Catalogs +=============================================================================== + +The LINCC Frameworks team has built the import tool hoping to handle catalogs +in various formats. We've learned some lessons in importing public data sets, +and provide steps to import those catalogs in case these hints help anyone else. + +.. toctree:: + :maxdepth: 1 + + allwise + neowise + tic + zubercal + +.. tip:: + Want to see more? + + Have you used this tool with a dataset, and you want to help others with + the idiosyncrasies? Is there a commonly used public dataset that you'd like + some tips for importing? :doc:`/guide/contact` \ No newline at end of file diff --git a/docs/catalogs/neowise.rst b/docs/catalogs/public/neowise.rst similarity index 91% rename from docs/catalogs/neowise.rst rename to docs/catalogs/public/neowise.rst index e8f30891..1a29ed69 100644 --- a/docs/catalogs/neowise.rst +++ b/docs/catalogs/public/neowise.rst @@ -27,16 +27,16 @@ Example import import pandas as pd - import hipscat_import.catalog.run_import as runner + import hipscat_import.pipeline as runner from hipscat_import.catalog.arguments import ImportArguments - from hipscat_import.catalog.hipscat_import.file_readers import CsvReader + from hipscat_import.catalog.file_readers import CsvReader # Load the column names and types from a side file. type_frame = pd.read_csv("neowise_types.csv") type_map = dict(zip(type_frame["name"], type_frame["type"])) args = ImportArguments( - catalog_name="neowise_1", + output_catalog_name="neowise_1", input_path="/path/to/neowiser_year8/", input_format="csv.bz2", file_reader=CsvReader( diff --git a/docs/catalogs/tic.rst b/docs/catalogs/public/tic.rst similarity index 91% rename from docs/catalogs/tic.rst rename to docs/catalogs/public/tic.rst index dc7aa908..f1fa6091 100644 --- a/docs/catalogs/tic.rst +++ b/docs/catalogs/public/tic.rst @@ -27,15 +27,15 @@ Example import import pandas as pd - import hipscat_import.catalog.run_import as runner + import hipscat_import.pipeline as runner from hipscat_import.catalog.arguments import ImportArguments - from hipscat_import.catalog.hipscat_import.file_readers import CsvReader + from hipscat_import.catalog.file_readers import CsvReader type_frame = pd.read_csv("tic_types.csv") type_map = dict(zip(type_frame["name"], type_frame["type"])) args = ImportArguments( - catalog_name="tic_1", + output_catalog_name="tic_1", input_path="/path/to/tic/", input_format="csv.gz", file_reader=CsvReader( diff --git a/docs/catalogs/public/zubercal.rst b/docs/catalogs/public/zubercal.rst new file mode 100644 index 00000000..f4982cef --- /dev/null +++ b/docs/catalogs/public/zubercal.rst @@ -0,0 +1,85 @@ +Zubercal +=============================================================================== + +Getting the data +------------------------------------------------------------------------------- + +See docs at CalTech. + +http://atua.caltech.edu/ZTF/Fields/ReadMe.txt + + +Challenges with this data set +------------------------------------------------------------------------------- + +- The ``__index_level_0__`` pandas index should be ignored when reading. + + - it is identical to the ``objectid`` column, and is just bloat + + - it is non-unique, and that makes it tricky when splitting the file up + +- The files are written out by band, and the band is included in the file + name (but not as a field in the resulting data product). this uses a + regular expression to parse out the band and insert it as a column in + the dataframe. +- the parquet files are all a fine size for input files, so we don't mess + with another chunk size. +- there are over 500 thousand data files (TODO - how we handle this=]) + +.. code-block:: python + + import hipscat_import.pipeline as runner + from hipscat_import.catalog.arguments import ImportArguments + from hipscat_import.catalog.file_readers import ParquetReader + import pyarrow.parquet as pq + import pyarrow as pa + import re + import glob + + + class ZubercalParquetReader(ParquetReader): + def read(self, input_file): + """Reader for the specifics of zubercal parquet files.""" + columns = [ + "mjd", + "mag", + "objdec", + "objra", + "magerr", + "objectid", + "info", + "flag", + "rcidin", + "fieldid", + ] + + ## Parse the band from the file name, and hold onto it for later. + match = re.match(r".*ztf_[\d]+_[\d]+_([gir]).parquet", str(input_file)) + band = match.group(1) + + parquet_file = pq.read_table(input_file, columns=columns, **self.kwargs) + for smaller_table in parquet_file.to_batches(max_chunksize=self.chunksize): + frame = pa.Table.from_batches([smaller_table]).to_pandas() + frame["band"] = band + yield frame + + + files = glob.glob("/path/to/downloads/**/**.parquet") + files.sort() + + args = ImportArguments( + output_catalog_name="zubercal", + input_file_list=files, + ## NB - you need the parens here! + file_reader=ZubercalParquetReader(), + input_format="parquet", + catalog_type="source", + ra_column="objra", + dec_column="objdec", + id_column="objectid", + highest_healpix_order=10, + pixel_threshold=20_000_000, + output_path="/path/to/catalogs/", + ) + + runner.pipeline(args) diff --git a/docs/guide/catalog_resume.rst b/docs/catalogs/resume.rst similarity index 100% rename from docs/guide/catalog_resume.rst rename to docs/catalogs/resume.rst diff --git a/docs/guide/catalog_arguments.rst b/docs/guide/catalog_arguments.rst deleted file mode 100644 index d6f02004..00000000 --- a/docs/guide/catalog_arguments.rst +++ /dev/null @@ -1,20 +0,0 @@ -Catalog Import Arguments -=============================================================================== - -This page discusses a few topics around setting up a catalog pipeline. - -For a full list of the available arguments, see the API documentation for -:py:class:`hipscat_import.catalog.arguments.ImportArguments` - -Reading input files -------------------------------------------------------------------------------- - -Catalog import reads through a list of files and converts them into a hipscatted catalog. - -There are a few ways to specify the files to read: - - -Other Topics -------------------------------------------------------------------------------- - -* :doc:`catalog_resume` \ No newline at end of file diff --git a/docs/guide/contact.rst b/docs/guide/contact.rst new file mode 100644 index 00000000..fa9e3502 --- /dev/null +++ b/docs/guide/contact.rst @@ -0,0 +1,11 @@ +Contact us +=============================================================================== + +We at LINCC Frameworks pride ourselves on being a friendly bunch! + +If you're encountering issues, have some gnarly dataset, have ideas for +making our products better, or pretty much anything else, reach out! + +* Open an issue in our github repo for hipscat-import + * https://github.com/astronomy-commons/hipscat-import/issues/new +* If you're on LSSTC slack, so are we! #lincc-lsdb \ No newline at end of file diff --git a/docs/guide/contributing.rst b/docs/guide/contributing.rst index bacb1a91..a2fd75ea 100644 --- a/docs/guide/contributing.rst +++ b/docs/guide/contributing.rst @@ -13,6 +13,11 @@ a description. You can reach the team with bug reports, feature requests, and general inquiries by creating a new GitHub issue. +.. tip:: + Want to help? + + Do you want to help out, but you're not sure how? :doc:`/guide/contact` + Create a branch ------------------------------------------------------------------------------- diff --git a/docs/guide/overview.rst b/docs/guide/overview.rst deleted file mode 100644 index 4418bfcc..00000000 --- a/docs/guide/overview.rst +++ /dev/null @@ -1,45 +0,0 @@ -Getting Started -=============================================================================== - -Installation -------------------------------------------------------------------------------- - -.. code-block:: bash - - pip install hipscat-import - -.. tip:: - Installing on Mac - - ``healpy`` is a very necessary dependency for hipscat libraries at this time, but - native prebuilt binaries for healpy on Apple Silicon Macs - `do not yet exist `_, - so it's recommended to install via conda before proceeding to hipscat-import. - - .. code-block:: bash - - $ conda config --add channels conda-forge - $ conda install healpy - -Setting up a pipeline -------------------------------------------------------------------------------- - -For each type of dataset the hipscat-import tool can generate, there is an argument -container class that you will need to instantiate and populate with relevant arguments. - -See dataset-specific notes on arguments: - -* :doc:`catalog_arguments` (most common) -* :doc:`margin_cache` -* :doc:`association` -* :doc:`index_table` - -Once you have created your arguments object, you pass it into the pipeline control, -and then wait: - -.. code-block:: python - - import hipscat_import.control as runner - - args = ... - runner.run(args) diff --git a/docs/index.rst b/docs/index.rst index d849780b..87f8d4fb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,29 +3,76 @@ HiPSCat Import Utility for ingesting large survey data into HiPSCat structure. +Installation +------------------------------------------------------------------------------- + +.. code-block:: bash + + pip install hipscat-import + +.. tip:: + Installing on Mac + + ``healpy`` is a very necessary dependency for hipscat libraries at this time, but + native prebuilt binaries for healpy on Apple Silicon Macs + `do not yet exist `_, + so it's recommended to install via conda before proceeding to hipscat-import. + + .. code-block:: bash + + $ conda config --add channels conda-forge + $ conda install healpy + +Setting up a pipeline +------------------------------------------------------------------------------- + +For each type of dataset the hipscat-import tool can generate, there is an argument +container class that you will need to instantiate and populate with relevant arguments. + +See dataset-specific notes on arguments: + +* :doc:`catalogs/arguments` (most common) +* :doc:`guide/margin_cache` +* :doc:`guide/association` +* :doc:`guide/index_table` + +Once you have created your arguments object, you pass it into the pipeline control, +and then wait: + +.. code-block:: python + + import hipscat_import.pipeline as runner + + args = ... + runner.pipeline(args) + + .. toctree:: - :maxdepth: 1 - :caption: Importing Datasets + :hidden: + :maxdepth: 2 + :caption: Catalogs - guide/overview - guide/catalog_arguments - guide/margin_cache - guide/association - guide/index_table - Notebooks + catalogs/arguments + catalogs/resume + catalogs/debug + catalogs/advanced + catalogs/public/index .. toctree:: + :hidden: :maxdepth: 1 - :caption: Example Catalogs + :caption: Other Datasets - catalogs/overview - catalogs/allwise - catalogs/neowise - catalogs/tic + guide/margin_cache + guide/association + guide/index_table + Notebooks .. toctree:: + :hidden: :maxdepth: 1 :caption: Developers guide/contributing API Reference + guide/contact From d617cd4c19e6127278a6c7e025877af1461f31b3 Mon Sep 17 00:00:00 2001 From: delucchi-cmu Date: Wed, 21 Jun 2023 18:48:36 -0400 Subject: [PATCH 3/6] Checkpoint - allwise, contact. --- docs/catalogs/public/allwise.rst | 7 +++++-- docs/catalogs/public/index.rst | 7 +++++++ docs/guide/contact.rst | 2 +- docs/static/allwise_schema.parquet | Bin 0 -> 136863 bytes 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 docs/static/allwise_schema.parquet diff --git a/docs/catalogs/public/allwise.rst b/docs/catalogs/public/allwise.rst index a66e5039..ea67ec83 100644 --- a/docs/catalogs/public/allwise.rst +++ b/docs/catalogs/public/allwise.rst @@ -18,7 +18,9 @@ Challenges with this data set - The numeric fields may be null, which is not directly supported by the ``int64`` type in pandas, so we must use the nullable ``Int64`` type. -You can download the :download:`allwise_types` CSV file we used. +You can download the :download:`allwise_types` CSV file we used, +and the associated schema file :download:`allwise_schema` +with column-level parquet metadata. Example import ------------------------------------------------------------------------------- @@ -45,7 +47,8 @@ Example import column_names=type_frame["name"].values.tolist(), type_map=type_map, chunksize=250_000, - ).read, + ), + use_schema_file="allwise_schema.parquet", ra_column="ra", dec_column="dec", id_column="source_id", diff --git a/docs/catalogs/public/index.rst b/docs/catalogs/public/index.rst index 389862a3..69f9ad13 100644 --- a/docs/catalogs/public/index.rst +++ b/docs/catalogs/public/index.rst @@ -5,6 +5,13 @@ The LINCC Frameworks team has built the import tool hoping to handle catalogs in various formats. We've learned some lessons in importing public data sets, and provide steps to import those catalogs in case these hints help anyone else. +.. note:: + These are datasets that our team has data rights to. We make no guarantees + about the data rights of others, the quality of the datasets, or their + availability. + + Further, please respect the publication policy associated with the datasets. + .. toctree:: :maxdepth: 1 diff --git a/docs/guide/contact.rst b/docs/guide/contact.rst index fa9e3502..3d076031 100644 --- a/docs/guide/contact.rst +++ b/docs/guide/contact.rst @@ -8,4 +8,4 @@ making our products better, or pretty much anything else, reach out! * Open an issue in our github repo for hipscat-import * https://github.com/astronomy-commons/hipscat-import/issues/new -* If you're on LSSTC slack, so are we! #lincc-lsdb \ No newline at end of file +* If you're on LSSTC slack, so are we! #lincc-dataformats \ No newline at end of file diff --git a/docs/static/allwise_schema.parquet b/docs/static/allwise_schema.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d09274f13057442bcdfb31e3e7b4bbf06699fcfb GIT binary patch literal 136863 zcmeHwdw^usnb+`!gw=I-VL@>>i{|*4B-8cineOnd#RW$9i@3tEzjd zs=K-?MrR2jgb+dqA%tZS5m_XN7!k8$M8qs2BFnNY%d&`wh=_=Yh=_=O-+A40Z=HLp zu5;`5Kif$rb?!O8`#sKgzVrU>v^5*KBpEITRQcYy(T`a%cLLGgBz5!`=~4LrNG zt1x=QMFx7eeL{upDX@=BuxmdF=EzO%++wX+SAcJR!a(ms018vTsIj?SFP2ZNmK5|$ zpERJ~zt~1!SlrB)N>1<#7aPFu{DjT(IgPq%@6Y~(0r$!uuvK(^D^lI6Zz}vg_6H2m z=l`G$y{HudyLF*|_y-N>XMfU0p9kogs^y1%(tv#Y5822IT9JBjWi_hseD5DJU?2G@ z8+(2$>SFlLpE5un`e_?_Q7hUh?kzuUKtK31Hu^k3JH)-|XAH=@FR_sqw4!ctuerp4 zz5P-fdwwhC68B>h^roM+p%=Aco#MWCse$kfm)Yp^0PPU>ou4%zue#huUeJoU#eMBE z1NK7&dwwhK68ELc4bXS~uub-&R=iW(XaBGP{nj6`(dPl$A@0+E#DIL|3LAMrEAAHe zu`3MN7yhV?JqlV`R*?mXkPutRAk(VeEmuT@&iDMe&c+=)wZjOW?%ec z2I$*=Tp?P|pV(57&-`%%^3_k<$n#;?;DrA8(+21ZFftQED>m$Kf=?3z_}M>UgD-;S z(n0#6KVd*W{U>eoCjnj0ci=@+JODx zK^uGCR@FI1yX~L>dH-i@Bx>3XzTqpz_EYd&Xy z-*rfZGk@%aUwy~`zjHu^Gymp^ z{n}9j_|CLSor7J}uSy$UaMzehJ=Q_}>M;ZO_KZrMgI&}=Hof4Eag}lutM(<{wQMh~Jd8u|H8LH<~B#1+LYI_a+T*xFu)X>$$Z` zUA4FmvIgL_Q#KIR1eQ0p)fJaFa|Y@ge4mJ?j)hmIN0 zPan6@xSZh7)Pu(j$j9bvBrGSmHFfu#0egDh#=`1}OH;Sc8=wy^sL+^8)|(a#(0dkb zC|ikep|4#upzm6;(YO-f(A3pS2IMWjXd_`I!mX(fOxPR#l8uFh7ni2q`9%ZStG}Q^ z<1SfW|4RnwhYFM}Vz|&ReZhc!|1aBwaS_9zsTcmT0r}<$8wraTZcROP!hn7Cuh>{v zvvFza$A85DeIai{!Ch4I>S+0N-hh0nU?ZW)1H{qCXx`Ebcb z;!oNiuB|#Hy?N6By?4uo zLgx&Uo`@(t>~&iP?42iV?1ikQI>1+*G=Oib+h8CJ(mJponXorCR4g_*2llZ zSGt*gre(mszN=!fAvowhv1`D-yk}!CWKF>Vess?OexYrHq1$pZeY$PHKJ!HtOKn=+ zg!;i34cI6Cs*M#kt%C3Ss|N66U$Vj6rd90SUov1%|1}j$ZCb_N{?`oHAN+M2D{NW? z-~87N;QLQ0yZGFwRqzd`OndV8{*nVt&00lY_e&=9-B&4SVc07CnyXCkyS}WzxoNBD ztG{eQ-*L4AO^sVcpE4d|xa}(nT9~&A|Hufx^%@1v4O~Tk_!SfDH-FWErY0U!-1hs| zn9w&~tDuFEtMGTfYJ%VJH3iPiTt&ZgtqFbIbq+K&bQS&9*G%YZzOJB!sjKiet~0@} zzFvWITUXJqf8B&W^$iD_+PaE<^?DQfBNZ)dU4_5=4HNvs8=S<^BdX|^ZZM(W|E2>? zZCypb@J$o?-5V9Ouyqyw+>IvqJKs{^+}2g}v)?kI-@3_xrnVkeZ2jq*Oz1bht)PXi ztMDhkZGyjkvjXR~uA-l~*@S-eI}S9pbrt>N@0idp-=d&}t*h`yZ!y7N`mO@!wyvTd z`K}55!mSQ8wRIJJ`c@PAx$h}xVe2aV;qRH?&)%lMxnEb&58Y-$KmE%NG_`dV{opT~ z&`;j3poOig@cVB!!JqgQ1+(YOCi6Z&EGD50=*6@KeoCip{t%Sjxaql&)eZ<){! z-t9nBTUXIH-EBhO|F;#iuyqxF!{0W+@4H8Vb6Z!@*WY78-}9>uG_`dVeeJKB(0ARd zpoOig@T>1N!SDP#3Y^=zioWXan9#T1=Ri|iKdjjL$42z6-&fGW)>Zh2_nC;_a=!xS zwyvT-_`V5!)8BQVsjaK%_wF~LZ+JjK3tLy=@BCd8{QAGAz`0*n(QiLsLSOr!15IsR zMZfv?Oz5lszJeCEuEJk`&;-Bg2MV0qx{7}7@0-vcJJHnERrD)AFrh#E2MTRr>ni-E ze_(=t@Q?!MwyvUIe8_};?;kqQ)YgwEw*LG-G@;*lSV0S0SK-e-Y=Xc2j}$n!brt>0 zKQf`;{GkI)ZCyn_^+OZ-^-9I*=ANh3!Eo@zd-}&n%_>Ue};M~?#^c|0z&_Dba4m7oO6@A;kFrgoM zLO}~#SK+rjVS@kQUn+2J>ni%@e`!KL@T3DxZCyp*_@oJa-@j7O!q!#z_5aEQzxOEx z&TU;qU-y&=efPgs(711uuc$lu*ZgY}_>QL)FxV>W6>O<%@28$Np>O+*PPDp!^pO#L z%QJu$=h@(-mAWDS!Ec!8UiX^{UD#x+mDC;ix1KR!uX@&rt!|uD9K&nBX~KS}V!<&~ z*Gkn@^`g>C&zj&L{FXwR!}Gfe^%sB31b_cI1rCQNka{_*Z5$rnt>mws2F3wxAKLwV$P zP1xsNbYh)`^6-l$>{Gv|V8IP{7|H{`XM#WZQYYMDDEGf)f@Rri~re#edjHOE(|d`4CUFkOz^k=ODEi6 zD9`*C6a1~W6*!z6>NJ$6-!|g^tAanTzOorn>+0$MYQnzujtd)A*X`P%>O0>e_T0)F#D6Zoo+6fg{=+^T-< zf1A)Bt7y1o;867|ADN)<{U3!eKpm=n{{NVuZ+@&mVdUsi^-~|4z_0yZ7uc!l$N#Sh z{PHQ~o<34`s`}AWXB$oxyzom37=|KlRiFMP6Z*NU6g1qgaH#s>t4z=*zwCrMRDJ)K zP0&ZLR-iEUa;f^Rt4-k3UvYt*s=oazCh$YoxWGWgFm&c_Rp0nk6Z*bu z6*Sx|aj5$GYfaEQzvhHGRDIRgOwe1dQ=l+q{Y>Tf9Ei<;GZqj6%rte%8_{dyz*1_ggX{RGSd8x6&Mz5Wdo`qVc8Ek;fkZsj}R zuijvSe~fT(G4q022l|z7n$RDrQ9CYLJ5lYxzjUJ!|1E|3#ejz~XXh)vWqQS_Z!533 zXhp-qt2Y_(H!Jvy0k31>)o+_#am{y>S6sBNVd0INjrdy>{KbIRvGB%sOs}~9yUHuh zQpXu@-(tdj=%nC!+!G!F)SftzOW>X;v#?MoBSt6T9H4+Q;3Gbo}Jy=gV;Ghm_{sN zyxB2G1ZT*xj4dJ_A_`-MuovMHBMdHC!j9{~(s(PUR%gFqIg#GT6)+l2L%>9Q3<1N4 z7XrraAPX3cN+Dn(&V+zrsu{*~+hck>4L)-(Vu(NBt zrcgMcSG-hX|Hk+M_Pg;nzJcHI7q7-!`|LNrhtCr7W<5*VcjZ}Hz5ULf(EIG{r?{cc zQtZw-`z>#gv!rq_Ts)9By~P80cUwG=x3Sp+dB2(&55F-jjD%SZzX`2A#2d~cjam^m zm)TpyU1U)LUvmM!V=SIg6u&#nG%IfdGtTJ!UiK4j?23nQ2Uk3d%7<`IRy>T!hj5ow zJdE>SP}y*UlRbsXR(|81Q1KjZd2*J-)k`Lq<0hsm2&@LOTZW>Xa%a+6j$daK<&@iv z&T{NR2n|*l)IF&{&r+yq_Zy z^u8NQ*f-c%8k>*(4tC2-!qj)rn_%J{*!pe0*ljJA#}*oGKIn}q@j`k>O1uMIf%*=- z$0Xi?dGmq}CJDH&%=-n5QfeWnyK^>jU zP!RMEiO9eeH-Vr&PG%?wdKZIbKy?A)M1-Zp)+SFvCZI$nOidj{sR&A@qbS|#lW(QG z+eO)Ff58CVT9ZX7({@pI#$FVqyhfe8@+9X~rYNtnDC5=6L|$cz@+ylmUhParQOc_< z%6OGUDX%g`d5t-F@)~#Y z%9ET|nWDVPqKsEN6B&>x%Bw8Oc(pSfMJcbcDC1QYrM$`%<#o`>D^GG>Ws34Di!xsA zOk_Z&D6g_8GAWof>h%MxOH){&BgqmnRghq5xz z9V>+Sb|uT=yO%a$v7N}W`Sze7g=(}VrYT7{CJE#Anjjog2=nbPL6|0O!eV<$5XM^B z5`?i*Y{E1p3CAU2+^!LX;|gKEogxU+q)k|CcW@-#zOa8sxnIR?jTrY^xLv?XoO4p- zpL0^=isgA$0?dT2ma!CFK9@hY_{@VZZ?or^fYl~D0k+B#VmVphVR=~ov|mM+Zv{H4 zzKSldidV7aQh|r%Px;e+6+ko+7cguY>BVPwWQ}z zwWR0Rw4~=?w4~?gv*iAZ>_tpNxLML;d|A?SKw081f-LbBGM4ljBbN9G3QPP&fhE0% zeIRQU5UR)uB7+St;A2PR^lT#EAbbNmGmBdmH3IbN_+%ZCH^9+lHNn85W}0Gz~E#7)w3h)vRKOij{r5KYo+)J)QIm`u`Z zTujn)AWYJ0WSGxgRBUieC(|NTU@Da|C^d7&E_y|}?dXG>@e1s$&_Oai>7h zts}}H@ex*#^d3i$_y`3Cx9RE6|s-_ z3DHOV#N;Esg76VPQTK?SFnh#TTs`6^kRI_9Igj`Xl}G%<#v^`$;Spca?})!}cf?Has zQM-uGFkQr7TrT1-5Etut`#7A%%;wu^r@fCiC_=-0}{05jISNn=6 zLwtsiA^u{<5WhiU$cO%;!Vteud!Q5ufbY~zvwK) zPdJuKAF?hzKrX!Wy3wCGaNzRKTz39ezE;XNF8j6X&;H_-5Cf*Xd!o3xv0be-uKfI` ze({S}uER6?^ z0wmZwG004Lx6EJdeADu3d82f~`Hb;)qPe$)B(L1Atrj;+jEn8+)}GAREUcA_P2{dx zZstq*W*&-v@yc=yim%oxh&_I6^s_@(eo0}*$;FkWGF+6b<(sRUHGGkS9e3s#@r9cE z(r17@E-R?ElOTKMqO-iQnQtDB(^_!IuVg3+=pY2~bUG>VhXIoUUFyV=kAo+t6%Oe2 zQ9dt{^?ZMbRv?a1X9bF<0jspKk*6)n*$~9jfQey*rcTMk)4+-8Br9`y118qm?Uai> z3!2cLh7f%gFri|tS??qzp1O#w*0c^QBOaUFRbz9zUM!zjEp<|JWVuNC>m@pzWN|ZJ zDrt$TIl9t=qDc|dKbn{)9xy?=&Y>l#<^)gF7Rc$kfU2DpJXyNn;Vo|8NN-hji%jBa@N$YbcPLk6 z1Wabi~lrt4HVHrd|gjY$#9W7!|LOq)jI`LGJ>VZ1Gjfl zR5Dx?&qSrw&CS}0jq*+>JvAplf)Kw|GsY87U4+hbhzDz!or3X)L6d^;`5IdMY2d`P zwCKa2Nx>HYG;;B$ffLix>Wf_P5d)1}{Ati~#kI8hA{Ty#K_i%W9JFNksRs=?@i=I5 z_%#U)Iq^7Xa`?dt4LR{RXmSzutqToBCC5edOuWDN2*-(rZuice>mq!HU90%8helZb zjStLm(L5tzl~0Fw6BVj(5k6C4`R7P9bd?+z%`*~K`TmJF(LM>wzp0|3tK^fEw^{q}Z#ZwodGatF(S7$oOu`li5 z^S}whLWdVkB`bKc>$)=<_|2exN!eJ}=@n#T8G%cPA1KmM6wiYuNI#^cAt;{*PEfHA zXM3dE;KHhcC#y9b{8UswOb0&?qBR{mBX9|YiMX{Yo(D~kem_fNI`Vno1ieiM_uczt zIx%h|F1?D!{g6^?Iw)g*ybk@OnXV@LWjgR-8*PUrp9if}+|Bo*sbmFDR_k@}!*KoZ zIw+;qbnJ}4B@`y&Y8~_ocm0qNqfu&1RXh$_KKemE9X(KWf0iV)YsB>XdHqo*d{ zAAWDJ9|Z9;S{i!danSWgztg3o*RT46fs?L_N9MAN~4cKk5%7mTL4V9tSNS zJy55k*RT4+PfF_O^{M{oH#c?kM7;*BQ2ex+hMssFI5|;IF8id%>yVGY3F_+)Kc&|f za-yEx@`>j`%SXRh7}3F?Fi7GXf{+sao2nFxv{8sMMfKwd_m9{qaEf*=3DGpr01) zivdyRt*2=#J!sX^kD+SilsXTZBz37#jcVfsTIy1DB<+l# zRm*&>mZq)rz==u?x~!Hu;l9)+b*V8gUrRHBmXfaRoY2yws_hR^RIQe#%m`Xe>QZ$i z?To-l+6r`85c9QtQ7taLXjN-x1T80ZsXEo#8G)1ZR4wzheNip-wOTo)&V!bdx>OxW zJ0ozCo~mVDxG$=uE>$aM-&9L|EscWWYMgdP(5hv=R!fsAo#q5i6g8-&DKmm5NnNUr zq@58sNn3%g>SVsQFRG<3RV!!TR7+i|j-;Itv}&10)zY+;9yn2{L05HBC)}6Xq%Kt_ zrkxSAl+2@QY4%UG)YochN}UHSCv~ZVUgKJt5jaVzKrKx>Cvc*uK`l+05j08aQgtNl zjKE3S3UoD2=4<;>pVZfC(v%rNlcX+HN7Bv+ zoTRNlSL0;9wlDQbeXUl`zNwbFR2@k>BWTq!U#q2QD?MQZ$i?To-l z+6r_vPUdU-QlHe)aq`r2@%h%G3z)4aCYH8XzffGdyYH7-hph;4fsv~J<1WwXc zpsR5*U)z`Zq`p=wXWvvyU8;_xoe{KZnXlE-w3Qw>QK>;!by8p3m)fK*RVSvM5ww)d z*J^3@Pqoz7YH3QH2Q4T1TFup62!z6)2Tl;wx1~GRhFfMVqc0?NM@lHk{v7Yn-lL@t zUDBP=P5wx&q`f}NAG=7M`Niw%#z}3r%pM0%Zmm>Z)x`M+>y$ZtA)4RSiU~wnPG5+Y z^PZyetf0war>|@L@_FC{WgB<(9XJ%|PvfU=)-`@RBXB8ojXxTAuMx{5UMtHGcU#XgP7Ts_pshjJ}Z6HGWF6 zFFk)u*ZA2(mvm>;^T+g!Up#h^I^&){rf>Y>aq#548h@;Rji0_++xV57zDTNX{FG{+ z8b3~)bd6s=4_Z!~PiY&!ozWMPy2ejQ_NDR1b&a1rbV+wcjX$n${Nk~T)EPJaxW4g= z$H9~HYW(s3HGcYPZR1yR`XZ^m@l&e(Y5et-&4{L1$j1Ro3Ga<+&il&8!IO(=j=TXm z_Bdd2Cv~di4}&J9Q{zeYFlbUb6`t&i0(Tol_#%yFzO7^gEFDCNZg3XCOmx)jfYCs?l6wFLR|;0eO=tCk>t9yGzIG(-2)S7roG zQfr^l{@7nC7er_tyf$S4^B zlibRC(NuF>M9*Ytq~=9W$#9W7)8&!16ICywj;!Fx!jhVSvLiQO;w#=VJFyXOn zk$|!zH(=uH-ZDF~f+y>c$|*BYcH{<3TsJL8{}C`5jU((wz}R-D>!Xic(wvF^X>2u5 ztk!fNptmyuB)8J6Mh7~jl8;@a{(oHE7ofU}^>V%m@likjsV`o+v0A9->wEC~%B{V8 zy}sE(`a9)%V|BBJPlrD@@VS92zhqm9k{4xp2QItVnw_1R|9qpkQm*DNyYtk!hwyJV zWx?MJ{sVs#mKC=W6>HS8Dps;WpCuBG{1!eOk@;iEG)u$3^er?$V_7W=aPa^1CGh_g z1ph9={}Qdm`N8$Yg{)l#7ZNltQ=8k)&*yfQ(#KbkA1K=_j%_TL#x`2?y~**N z#p>3^Vr+)x&(94k&d;ut(jzDr=wxntF*>{s^4F;R8*|5JR|n#mTExk_tt-h_24pEl zH)kF?5jimcV))qHhst(2ts&83Ce6?n(MqSW8+@MIEYvS}BqBb44! zbz`HDo?VvuPLA&Z4(V_^$*DT47Sh8jrPSg8w}VJwAu*`(x>ViWSzMTj=6vu1dRQ~) zh~??(xVqFRL`Mg5YrGCS>)6F_WTTWG1>2upk@hn-(8!D__LdySIuSbPu(?bvx3v@- zU!nYl%hUSOeAcVZkY|~%vmsZ#Xl%%(OJ{qa?R**9EcgW5&bcm3^@r`CSRDf)6T2lTMa&`FZKj9!FPDs zIQS{9zwqiTR%U*zQ$Mv*JY*jbD>J{W<7F^pUe*e=**!0xy%-%FC^Ea!+45GQCi{m@ zKeWEFZTBy|`RhV#Zf`LGT9+B2%9`uGDW&W06vO6vk%bJ6`tW z$PcmYXBVKak49R>>X4UDin+_^^wk$%Su5@qql2Dp5Omfq)H0OsPh+F*!vIoqWkmHZI(l^g5cPrA#5*e(a84QcCgR{FS5c~X3o&bQ{q z6dMe$E>*kMw8wqnX?xMj*H;Huz2`mZJfgEdJgSaids*v%wKK2rX@Re$>FlQuW97DC zj*R2c(AN=goM7aswjVs@%sj>RgQr;|Pv9d%ue&0wpW@oMmu)-pwSFkzb_V+l3$cVZ zKh@FYVtOFlc@3=RFGhx21+VoNXMSko0lu~>^`YrU*jo0SQ!0amU4v0W2>;h_Zc+fmSQ?0_9#NFQ-!W(e1Ll+iKf_F()EY^Sqn)t zje8L3Ncj5?)ONb?g?l6W!W*m|(edZF$#=-LntibSShmK-nv%Ar4ZygAjaN6q_iu6z zXw%lT2bjOQ^c-HBC=R%3Yx;+nm}f{mhv(nib*Isv%Lc;ZtyNIY4ruh|$k)|rfA$h` z%iup(mSUdk_G>Usfpxc0SOxRk!|%sl!rsz+%WI8X=7puFrG=G&h3Haa$!kruIu|dE zqMnMLhd`UV(tLmjT>`F^pEub}I&Fh9Ig*cfgsEi8K3db1E6-d>3AZj{2?|BPkl2R-LY zun)A&=PjX)6LG)JYd$IS!t>>^Y}KTn@qN(GxIsTWFHHSJP5R00gMM-b{qVdH^^;4R z^fS8;`k6K8hv$W;pDc`(xWDW4)38qsXCR{9>$H8D-$R=>@&segec%bs*BbQH-VdH~ zX(LYq`@s{~)7nBbvI1v?wdZ~_)TVOFh56Bzw_OFjE{S%`XfDKZ;(U{by?2z;^}YQV zhojwh(;e7;=(>|MyXa0=EAx*PW!GsM4%L_F6T+-ZSer79b#U!UBcUU3WHow7EF%y|38M z@v7(fGL)sEx;JI#hiL2BYGlQ8zZ!P)rOk!O z6P{s;va^=j`cv{lY=1bHFtopMX>(!ngy%*0T`1ky0qRfYhiL0L>5H`W2W#xy=ECHO z-beEGpX@GD=<{mRc@Y?6Nt+9^{Y4{dg_z+krIH__tq(|Fr0xG^rOk!O6TL_2bw0ZW zXW?Kz6n_5)V_<&9NZ0;y>HX;cpni<~pOPP@t~`vn|Vm6o&Nk7204T4;O5 z%i(LMk^SHe>`UrI*YE72-x9os*dFP4MbOpRJ{|9f`VQ01=$J*;l}=~uzV#fp#V~n; zct++nfMXk3-(mLIYTMK4tc`twsP7Q%jIPNPL|tk7BQ~Dc58l+ar)>i`=GccekV|*- zwb+O6hn>Ot2i#Q`b*0l8Tf33%H^er8eYw<$t_=*R^&Mgxpz9F}ZNoT`^{wH@++guj zmJP8QVDAscWphK}^M-2=5l&a@H*oC&>f8GqFXRUw3lHwJu(s#D-pl*e&||xZe$lXY zLif_3{f5sQYkP(^!1~tk?F{GGMmxhf`hM7%=obw-i_y3a`_Kk*YTMKKS~?%!2Roy4 zUqM&fKe4_wbUTy%qPO2*bKrgOwRE0qYy-S+4c*uBIi=SSDjL#_zcEy=^4)$@A=`lmd61Pah?w6Md=!XXDnE`Ub_#xQD4jH>c+Kvy_WGF z;&_A4DGfTq{prXMtM4%Tblxu-bSCDd;q%7kl!i9I)@#GJGd8C*=#2NRq1&107Y#ZS z^V0BnW4_kV23X%3x}EX$T7%A5-x|K1iGI;rXJTHu54?5rwS2ucbUWj7O7>k6?OcHM zt&z~rK=Ae2ocBJuSg#GAH#VnabPemZoc9pN8*EN#&>8PrS$&7ur}H%ggU-ZyZTP$~ zUu$RstZxn9&e)vNpfldLhJFq~^os_a@p)md`@Z58C$Pq^}QeKwca`t>$TzY#^#iU zHo(_wL$@Z%8g$0zrTf4e^|gjJ zz}IU-w=+Jc^wt^gTSK=q(JvZwCg!E#^TyU|4Q+s}*M@IrbiLLyMjt;TWQgI~rxDK2s*Ycrq~_&F!hD=AK@>xYm2y#6SG3U zuam7BVvF)Tot}dJrU~3P++Io>_H3%s--bFzD~uW9EmWny4V9;Sb+(x|`GqR=Yr^M- z&fB6XzAxiV_}tKWTbdtb{fu^Re7uRem1V;0CvpB5v%G!1lCNuzFfZPLZ{J4w`Nr+i ze56&L8y+YxBsMZ@n;0XmQGjo(=GNFb!$L3L8isg7%x@Xm0L~r77&c6OF}j93z;xa# z#;_sUnJH$iDNb|X--&3;F{swobJE|2X-{$ts^_UG{B5{ADdSHMzEqzZK5u;dY2Z!x z-0*pWZ?<`l?P0$Nbt}t+*beF5hGEXHYV4T7!38(sd;< zh7Hlq1`Kn4k*CvB;B$WX{+4OZKOp^WnD&&GV^H0E2i7sepYy}oXty~(@TK_NaP`$~ z&d+%Zf6mYKW}Nf8%Y@es$2SY@5zp~fvAVL8Jr?$QAo7&)ye~_6S_}KGEb^4~ydy<< zYKMK-XJprVEm!90d?@VC1Rq6ju$$%ep;?!adEvFQmG)vRu~&$B?mfke z(Oj!o9S(m@a$&r^G#}YAoZW-wUY$v|-M?kXQ+St-A#0 zD!tF!WZ|y5$Pdrfi@QtFxjjR?YBfJJ{{;Ji_{|bSY;Pq$JUuZxwN_csrezczkx38Pt6ap{mpi1e@cFM?GMjHwu_PB`qBdX4ywkF zc9aRPF2|QKmVYtk8K-Y=0b?33?#Er&M>I_i7R=-FXXmf3P~(O<&vHd2;P3 z*XEkVv5n=@*hXvVXy$MxiT9Y)^`po}DLuNrxG=kdd(ws4T%#~H(8!E6Rtwe9CdZsy zh(cU-b6s|n3C+%`TgwGJw+V4%jdiLo7n%>rj}AlpYsKC*drci>!t*7YP{+96)UBSi z<-)7$<}${FH`jAVnegg3GTg-T8C~mHS1!D|_B;>H&s~P5FYj+0hy7jeO|Nr4hrQP& z0`~v4T&pxXyj7@r#!PD$$7Yw|oZotAJXKc)3db^@d-nOU*=BZ)ossIB5151gC^^1X ztj?{J((rvo=%>ni#3zGtYJnscHkTY^m&=V$u(;` zn;aj;Z?@ZfE^QRyjAtPlU!7XDRG%2{!ESeb&X zBBoOLdKz?x& z9u_e@VnNzEj_JW9{G6PrL;lDFWX)kZ(Smepa{}JqNNys%lkji^(<7M9Rkon~`UJec zm|TaVGmy4MFda!kp$Q8LC0Y~ka0b%}Os6X2@cw!d>T9f3gNIp6k70VMRfYG($D!zI z66$NT0%>a#(+87KKiQcL$Ui;~?_a`n3e(mIrXxxCXJQ7JNDPd_!&yuxFr8Y@!26p? z&}wF84IUoH^f;!cXI3G9Bm?i?NJ4$3F>MWCI)Uk&wF2}i8F*DO3H6t1LE6e(c>g@6lQWP`4WyyaZUPF8TTls!3Z}D|o^CBb zes&bzUrPY#bOq8@9Mefm=Vs>N*ZL?FEhPXtVnN!P!E`(U^)=C&gZB-M0=;9HPE;VB z+8l*K+X;A>nZbw0F+GmyY3mr|1KXCho`CuqZ9&>@v`Af33(~1Lx{5t?9pmUa5}TNw!nB3y)FL+REel>UW`Pja5N+z1 z&b6TFB^%fjm$501V45|}!`M_NXC@#YED#zgrjsBM7$KT=%YynEhyQ>QF+GXt=?W@g zBndCsumF^vLDL+-bQ06K7MM=5g{D)npuR?dBQy<24`Eejtu*A%U{x<-I)!PZ051U! zfI3u7qJ>p6gXsjOQx#NQ1D=4rgDMjd$?VRdL&(<*wr*q<|L9Npi#E40IaUBC9%JQE1@ISj;n`Cg zPhT|#b!ep;?d3y`kyL@y|^U zX;tiVNKieA92Z#hOEkuZNf^g*z*%hvcfu&*lfJTb{PW41zt`}K}$tj_~RQ8C&{yOL*0T`#^I@3b5xk2rB z#Kup%@Ez?9RWVkCbw`MW2Wl-eI6dgd+R#)XSF{&!RG5%|^i`!ctPR506!e|v<&)O< zG0wlJFSow&Rf!m1g9ZOokn*;=@r&evUs!LZj0{PBp}m{=r2yb}b&)mSGPE?5FKH!2 zeR=Uq?IR}k1FVrcuiuO{3oV`bGiKi&`h|TJrr%P7mxtbf)9<2R*!y7mU6uTfW6L)A zU#8zI+hYR#9+CX+b>o-m7x&Yseh*51Qv+T2<@$xaL8jm808pj+O{Qkd{4)J!`F^K0 zI?2mttYf)UFZzXjW2WEbCNEz}#5w&g`h~q+rr(p2--%;g_~rU#d+5;L7bL#}-S}nt z#q$DGzgBaM$yZ4Yb>Wxm7tTH~{caDi{tX5$YqLG+m!IdbM(ezM#+u7<`rJQz*e{%m zVftN>{3b06f2Zv(l5ie8yVHar$2RMc&5b89ZxE@|(=K_@$#5#sAW- zvv@2!U`-Yzy@rck$9RR(8`;aYQe`P$Zp|ehRRA|oR3i=*61#BT)=D&?Q8Rs71I&MV z)juW8&!6H;_IP=)*Y+{9KI!;zFP|O)J^?v;gqP1)^Epo6OP|R47&|+RL{}xh$q^U7 z+VK?h5AseP;k6+6X+{k3qO~SmIObe(-SLlH$`%5S{7!cmza;hemxPeYn$v z#nQZd5{Bu=d(b!Z$8g^O*&dtZ7sG?ITs@<&#!!ZLSyl$VRz+SoL{U z9{fSB!1QOfk9NLgEk>FBa{I7a++Kvf-TZR<;P)ykiB(=6<^yd`-#lK+^1Gr|=7{8X zRq|`J4;pt64Pd*;<*OtIT>N_2hkd6P^?z96Q|n#u+&&my*hjhvMk(6A)8D%3JMh9j zj!JxDB;v%o_08>LT=RddVvIrm&*@?RUpKc0WM!-<;-*=(CNEz}E^_*! zevpXTABp}D`+r!&qC8n`iBDx*c#rz!aqg_tVJYwQF_*sd@@9GLJ}X%OCy)LuV`Xxn zbd-sYFR2kJ>wQTSVyuY!Z}|OoHb@(_@yA{GG?rKJY<;sx{RhfiX>xj%#1OY{p)U`4 z+4tF}tmHN?53AK{oPW_D>g^*d;$~8wQe}zHx$z$Q;_*IVRlg|omB_Un`tp!h#4xpT zOCSl zXsJ<&A9Le9^u=RuTN9I<9`^s7f3Nu(^B*FfIJAEuZz|)G*TX(!9CfPdLpDC{!h7h8 ze_N$xos0>8n>Ps)Lh-q()7I^uLbv$>((H=bPL&QvsQl>LDzT(1r*aw}zEVLJ` zMw8R4BpaOnF8+gmSD}(BOL=o^E_prt2anr{`~2`X1vSOi_pCLrOw;%fs}9Cn{5u%X z|MT+L|HDg=oY_9aw>_+-CNE!s_Rr`Gc|Ghy#EzxaZ*53C9Aa?dJ?w*ji=~oEN_i)7 zXQ2mqSuEhFHFZSFd)OtfhkeNJ>Y$>n#E}Z4m$A}QUDiW+d&2oG+Q-ptxgI(acluio zdHFYvW)e|e9>@QjzVMfN`w-&~dd|a2RwaJIjrVF#^t(6_D>o_SZTBXx_|BGfNXnav zyX5t-4-v1HE?QWV+hRS|+KfZ3q4xQHp@+VB4Evcxo|nh{zm=ZsgN;8TR;IwqXROMU zb3N9>KE(JVLhWPH#y4De5C6f(AIGh@ls8rEO?tvcn^+lLr`F#E6$ zN_@(V*V_lgD8z5^Myzz5)5};(Q@yOO!u}TAuUu%)SR317JvceT>5KaHun#f*V2i@L z5}&eMcn|yF;}0vfC*_@LyY%HDFB^XhK>LSDCL5oor=Srl{?fxf#Q1~m*;^wv{-_J@ zVIO?_F=LgtIlW3^Al{q5Wby2={j>5gK234{&Hh7zCR7^|CktWZ4Ns9rH6fp@dvYy1&L2Ay6_(M!N(s~a)OtK@#!(nztERnUhxfd zSpS#uHl{fL=J|^lf55pq_?xovaTnghKKS?}YmIEn_0#NNZ~l_n2dwKKm@$z#_RRP#~*ELyw2%mtc@woe^29&thFoUO<68^J?uk_Kj?7_YuLuO zTzC)rVB-&vd38|u+eEwS)R%|6Z2ZxN_Rq^_tfNz6y_T(Cc=!)7{$TboX5-5)yoY_z z^^b*i);hT@*H70u|H9vT$jinb*-C0&%9~5M6_<2Ani zu?OdWczHPgGX+Mi_*)No`S=6&`?h)c3bcQxzx1#VG5(;-A=aA2!@*uB-a}t(`~mA* zho!ue$9j^NuYc@8`@UGjR^hZukCQB5Zbo5J6YOm$g5MP0)B2^*in>9uXKergRJ zcKTZndD-|QTS;v5@<}Vb#_0=xskaX?{$TczEKB^j8}HSg==fvL%GRa4yS>TF#~&lm z{)N1$mP=j_`w-&~wp=&94ONBW!P3#;p8PEve`IG8M|pWz4`^@o(jKyG{IP3|9pU9O zR(XokH}@}M{IN^zW5ULtbm6_)6SI#ID=Otp9q&zEw*CR$Eqz${ODlEKDX)ip@NZi( z&uB#@KIO)H*asVb?85#(rw9IEim!*7`xid`7=iZB%ftCUPG9&-5Bm_~4`v@bHa_mc zd$lL}{k!Z;>ZFu+>X=JkZh7%lcz!FgR)~S$N)&);Ha;Dh0;QvRz2v3)^K3khwlTBK z%U7WNoBbU%6~15G1ftu#Jhp#MU$lQWzifYs`gi1Qye|3O=*BO;3f8v^^SPZO{N9!P zrmQafiv3m??_kxX_9VYk?JoQ>{nGu*C=6#4M;b!E>8UR3txUgk{0>B6{x8;B6C-#E z3?2!4a@zyfFN`;$!+TIH%FE;YpVK$%m-Z*9--$`dZ><}@Ouwu@0p8n^-&CdxzjD74 z;s^oJ)M3f*^sz4da{bP8`%M;_LcbYY)#`4)w7mjR>-aV=5Bq<($^(zO=$ExuD0YyS z2mP*b`d##k^Ixjpy5x7O8^27y15l^ixc3&3{HA8Q@JqXX-rjKkK6zw^ulG&2d*TK&tb$^te4`=)RFs3Q7vsvXK z<`41wZ>u+XId-JZ%ftEK=`QQB%pP||`z!6p_1G4t?`4m?{ZW4b?Jv11`5kX};g_Gs zSm5n1dqncP=i;|xe9!EW&i_OkvZihPK^NYm{h@!PZFgdauV+@Q^&zu=qyv~jwX{bK z-Ac^!@_7D#m)8&84#OJ9KQ`c@KXI8WvAVR78(4_VwU_3G*9x`SJY_fP0M+Qn+t+H~Q!CH^>S3UAZEAN@&x(LJ zw5m^JSiw^!5}U)Uz3}rFNo$Dp2Z;pS_JThurE)%%E_lFqHr5v5j>_UfV!0S&bz+Sj z1O5-T1Rihy!Jpf|QHE_iEn~H(k2&gd5amH}XN>g$?ay5wsYRqeD(aWpLq~f+vSUSw zr(C2`aMTF>Bd?E{G}Lw=5|V`rLj6MpRr1kzQ=hI-aml(zfE9_JH-Fs_Hc(Met&&i{-0=KYBWy>?+HaahtD6Ya%J56&|(dJ_dnZ?iXgdxe?Q zl%$tk>q$O1AIIoT*CoB--stUuK39jC|AhXk&F!y?eDj>%$Pr#1*8ir(dZ)qvq$4{p z23jtp=h*&q`WOsy6?-3(^tu0_O<(cH*pw6NOAE8hi`9(*xDVvSPe?rTUmf=cDZT}F z({@YqqYZ0(4fsEFoXao#joaQ?e_0*fTflXTR?5Qihg|Zese#${N?l)iE)KB?=saz7 zny=6Bn|_niV!f347rp(BZxp~Q4ud~A?kcbKCu#J{OO3_(5?fy#NktDG8XiJCDvtY8 zw|tcKBF?#^vv409{4tc@lJrdLx#Mg3^!naHEWzwAR|k4npBXFC4Nv_Q+_$RcHuJ~E zmy2oaYwNr|;r#D3D4F_S5BdxbeW`*^+4x*<_&ooRsx@r(-_&$)9t3s12X|6d4^H#(ZlWDG_czp^ zwh9a5E0o`nBfLDef9LvahkasQQhCQ$P+n>?V>Z6r4Nv98bBMDW(C@dvZ{hu4N&jRw zJTlDb+xBYXqb_`hzR}*&Htpn&TrV9t$o!|*c$+ruVs(_;xE1B)@&3;s*SCki!={>O zL!2r24idDFk+mZZf0>qe(|u_?zqwLE`NtJ=I{YQApFy`MW9}nW7Nj)!0NBxB=3=2n7FqBdHqx2f5@6{jD@z>sBbY+XK{g2lrzZ;TYkMS36 z55oUttlb?hUnONpes_^myvXn1mx_+-CxsdgWm9{+Jj_?yE`E`SI{s$!Rlc7EvDc}r zs|T6C!2X}xmzTWM9&vq^VMmVe^4R`C$+Uiq@^XI->-{^tJna8-`o{JH_e;3E$*Sac zBIV*&FE3l~nQzsq$s$?YR!9qLV9-oLk`*?t4(IXNNm88_ZT-YS=OvLNMcOmqIt>o2@N%)l8IA#bYK zo4li3UTf+Q^QX`**Ptsy8|@-5JAW+f`Jlw7+;|Up;TvT$o2!AQ}i?YCtBaXS6DZplL!%*r(QCmT;|X zBbj!=eCpaFgqUcFnIc4jg z6ELQ8v_ERA%6tr$VqiWtpQU#+89ls{@1Wq%msnIhf=Ou6{rv z5V1%D&S2w_L_OAHFa{5f&*=S$>gYgz9_s*J#i0i-`}wVW{p5DJ`Pt&u)@Sqe`ey4> xJLP&~b+h)V;m-|xZs78dzJK-=2M+xHe}7*CzpsJ)(!ha7e&ehI)7EU{{{b+x7|;L! literal 0 HcmV?d00001 From af8e6ec816cb2ee1fe4904ee1bf2d690b7f551ba Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi <113376043+delucchi-cmu@users.noreply.github.com> Date: Mon, 26 Jun 2023 09:54:26 -0400 Subject: [PATCH 4/6] Update docs/index.rst Co-authored-by: Olivia R. Lynn --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 87f8d4fb..9cdfb2cc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,7 +20,7 @@ Installation .. code-block:: bash - $ conda config --add channels conda-forge + $ conda config --append channels conda-forge $ conda install healpy Setting up a pipeline From 33f9213ec6b450a298feea6b40a09b1e446f22ba Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Mon, 26 Jun 2023 10:06:14 -0400 Subject: [PATCH 5/6] Update estimate_pixel_threshold.ipynb Resolve conflict by removing intro notebook. --- docs/notebooks/estimate_pixel_threshold.ipynb | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 docs/notebooks/estimate_pixel_threshold.ipynb diff --git a/docs/notebooks/estimate_pixel_threshold.ipynb b/docs/notebooks/estimate_pixel_threshold.ipynb old mode 100755 new mode 100644 From efe6324dd39febacc25d9c26e7ba859f32ce457f Mon Sep 17 00:00:00 2001 From: delucchi-cmu Date: Sat, 24 Jun 2023 19:25:03 -0400 Subject: [PATCH 6/6] Use an existing directory for sample outputl --- docs/notebooks/estimate_pixel_threshold.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 docs/notebooks/estimate_pixel_threshold.ipynb diff --git a/docs/notebooks/estimate_pixel_threshold.ipynb b/docs/notebooks/estimate_pixel_threshold.ipynb old mode 100644 new mode 100755 index e0adc254..078b039b --- a/docs/notebooks/estimate_pixel_threshold.ipynb +++ b/docs/notebooks/estimate_pixel_threshold.ipynb @@ -46,7 +46,7 @@ "outputs": [], "source": [ "### Change this path!!!\n", - "sample_parquet_file=\"../_build/sample.parquet\"" + "sample_parquet_file=\"../../tests/hipscat_import/data/sample.parquet\"" ] }, { @@ -203,4 +203,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +}