diff --git a/.github/workflows/test-build-push.yml b/.github/workflows/test-build-push.yml index ec7463c2..03858ecf 100644 --- a/.github/workflows/test-build-push.yml +++ b/.github/workflows/test-build-push.yml @@ -14,24 +14,19 @@ jobs: pytest: strategy: matrix: - os: - - label: Linux - runner: ubuntu-latest - - label: macOS - runner: macos-latest - + os: [ubunutu-latest, macos-latest] deps: - label: Latest - spec: "" + spec: >- + isce3 - label: Minimum - spec: | + spec: >- python=3.8 + isce3 gdal=3.5 h5py=3.6 - h5netcdf=1.0 numpy=1.20 numba=0.54 - pillow==7.0 pydantic=2.1 pymp-pypi=0.4.5 pyproj=3.3 @@ -40,10 +35,14 @@ jobs: scipy=1.5 shapely=1.8 threadpoolctl>=3.0 + exclude: # TODO: Remove this once pymp is gone + - os: macos-latest + deps: + label: Latest fail-fast: false - name: ${{ matrix.os.label }} • ${{ matrix.deps.label }} - runs-on: ${{ matrix.os.runner }} + name: ${{ matrix.os }} • ${{ matrix.deps.label }} + runs-on: ${{ matrix.os }} defaults: run: shell: bash -l {0} @@ -51,12 +50,14 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup environment - uses: mamba-org/provision-with-micromamba@main + uses: mamba-org/setup-micromamba@v1 with: environment-file: conda-env.yml environment-name: dolphin-env - extra-specs: ${{ matrix.deps.spec }} - channels: conda-forge + create-args: ${{ matrix.deps.spec }} + condarc: | + channels: + - conda-forge - name: Install run: | pip install --no-deps . @@ -81,10 +82,10 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set environment variables for docker build run: | pip install setuptools_scm # Install setuptools_scm to get version number diff --git a/CHANGELOG.md b/CHANGELOG.md index c49a750e..47fa5ebf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Unreleased +**Changed** + +- Split apart OPERA-specific needs from more general library/workflow functionality +- Removed the final NetCDF product creation + - Many rasters in the `scratch/` folder are of general interest after running the workflow + - Changed folder structure so that there's not longer a top-level `scratch/` and `output/` by default +- Changed the required dependencies so the `isce3` unwrapper is optional, as people may wish to implement their own custom parallel unwrapping + +**Dependencies** + +Dropped: +- h5netcdf +- pillow + +Now optional: +- isce3 (for unwrapping) + # [0.3.0](https://github.com/opera-adt/dolphin/compare/v0.2.0...v0.3.0) - 2023-08-23 **Added** diff --git a/MANIFEST.in b/MANIFEST.in index cd7184a7..12f0422d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include src/dolphin/shp/glrt_cutoffs.csv include src/dolphin/shp/kld_cutoffs.csv +include src/dolphin/py.typed diff --git a/README.md b/README.md index 34e1ccd2..ec5fd659 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,19 @@ High resolution wrapped phase estimation for InSAR using combined PS/DS processi `dolphin` is available on conda: ```bash +# if mamba is not already installed: conda install -n base mamba mamba install -c conda-forge dolphin ``` - (Note: [using `mamba`](https://mamba.readthedocs.io/en/latest/mamba-installation.html#mamba-install) is recommended for conda-forge packages, but miniconda can also be used.) + +`dolphin` has the ability to unwrap interferograms, but requires the optional dependency of `isce3` to use the python bindings to [SNAPHU](https://web.stanford.edu/group/radar/softwareandlinks/sw/snaphu/). +To install both dolphin and isce3 through conda-forge, run +```bash +mamba install -c conda-forge isce3 dolphin +``` + + To install locally: 1. Download source code: diff --git a/conda-env-unwrapping.yml b/conda-env-unwrapping.yml new file mode 100644 index 00000000..51f90777 --- /dev/null +++ b/conda-env-unwrapping.yml @@ -0,0 +1,9 @@ +# Can be used to update the environment with the following command: +# +# conda env update --name dolphin-env --file conda-env-unwrapping.yml +# https://docs.conda.io/projects/conda/en/latest/commands/update.html +channels: + - conda-forge +dependencies: + - isce3>=0.14 + # - tophu # Once tophu added to conda forge, this will be installed diff --git a/conda-env.yml b/conda-env.yml index 9ca8f469..3abba6d5 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -5,14 +5,12 @@ dependencies: - python>=3.8 - pip>=21.3 # https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#editable-installation - git # for pip install, due to setuptools_scm - - isce3 # >=0.14.0 # Right now, isce3 is messes up conda's solvers. Should move to optional. + # - isce3 # isce3 for unwrapping has been moved to optional - gdal>=3.3 - h5py>=3.6 - hdf5!=1.12.2 # https://github.com/SciTools/iris/issues/5187 and https://github.com/pydata/xarray/issues/7549 - - h5netcdf>=1.0 - numba>=0.54 - numpy>=1.20 - - pillow>=7.0 - pydantic>=2.1 - pymp-pypi>=0.4.5 - pyproj>=3.3 diff --git a/docker/create-lockfile.sh b/docker/create-lockfile.sh index 643a9425..a13541ed 100755 --- a/docker/create-lockfile.sh +++ b/docker/create-lockfile.sh @@ -5,42 +5,82 @@ set -o errexit set -o nounset set -o pipefail -readonly HELP='usage: ./create-lockfile.sh ENVFILE > specfile.txt +readonly HELP='usage: ./create-lockfile.sh --file ENVFILE [--pkgs PACKAGE ...] > specfile.txt -Create a conda lockfile from an environment YAML file for reproducible -environments. - -positional arguments: -ENVFILE a YAML file containing package specifications +Create a conda lockfile from an environment YAML file and additional packages for reproducible environments. options: --h, --help show this help message and exit +--file ENVFILE Specify a YAML file containing package specifications. +--pkgs PACKAGE Specify additional packages separated by spaces. Example: --pkgs numpy scipy +-h, --help Show this help message and exit ' -main() { - # Get absolute path of input YAML file. - local ENVFILE - ENVFILE=$(realpath "$1") +install_packages() { + local ENVFILE=$(realpath "$1") + shift + local PACKAGES="$@" + + # Prepare arguments for the command + local FILE_ARG="--file /tmp/$(basename "$ENVFILE")" + local PKGS_ARGS=(${PACKAGES[@]}) # Get concretized package list. local PKGLIST - PKGLIST=$(docker run --network=host \ - -v "$ENVFILE:/tmp/environment.yml:ro" --rm \ - mambaorg/micromamba:1.1.0 bash -c '\ - micromamba install -y -n base -f /tmp/environment.yml > /dev/null && \ - micromamba env export --explicit') + PKGLIST=$(docker run --rm --network=host \ + -v "$ENVFILE:/tmp/$(basename "$ENVFILE"):ro" \ + mambaorg/micromamba:1.1.0 bash -c "\ + micromamba install -y -n base $FILE_ARG ${PKGS_ARGS[*]} > /dev/null && \ + micromamba env export --explicit") # Sort packages alphabetically. # (The first 4 lines are assumed to be header lines and ignored.) - echo "$PKGLIST" | (sed -u 4q; sort) + echo "$PKGLIST" | ( + sed -u 4q + sort + ) +} + +main() { + local ENVFILE="" + local PACKAGES=() + + while [[ "$#" -gt 0 ]]; do + case $1 in + --file) + shift + if [[ -z "${1-}" ]]; then + echo "No file provided after --file" >&2 + exit 1 + fi + ENVFILE="$1" + shift + ;; + --pkgs) + shift + while [[ "$#" -gt 0 && ! "$1" =~ ^-- ]]; do + PACKAGES+=("$1") + shift + done + ;; + -h | --help) + echo "$HELP" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + echo "$HELP" + exit 1 + ;; + esac + done + + if [[ -z "$ENVFILE" ]]; then + echo 'No environment file provided' >&2 + echo "$HELP" + exit 1 + fi + + install_packages "$ENVFILE" "${PACKAGES[@]}" } -if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then - echo "$HELP" -elif [[ "$#" -ne 1 ]]; then - echo 'Illegal number of parameters' >&2 - echo "$HELP" - exit 1 -else - main "$@" -fi +main "$@" diff --git a/docker/specfile.txt b/docker/specfile.txt index 4fae4e09..6923f494 100644 --- a/docker/specfile.txt +++ b/docker/specfile.txt @@ -2,162 +2,168 @@ # $ conda create --name --file # platform: linux-64 @EXPLICIT -https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.3-hafa529b_0.conda#bcf0664a2dbbbb86cbd4c1e6ff10ddd6 -https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h5adbc97_2.conda#09be6b4c66c7881e2b24214c6f6841c9 -https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py310h5764c6d_1005.tar.bz2#87669c3468dff637bbd0363bc0f895cf +https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.5-h0f2a231_0.conda#009521b7ed97cca25f8f997f9e745976 +https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h2c5509c_4.conda#417a9d724dc4b651f4a711d3aa3694e3 +https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py311hb755f60_0.conda#b8128d083dbf6abd472b1a3e98b0b83d https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2#a1fd65c7ccbf10880423d82bca54eb54 -https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.12.7-ha878542_0.conda#ff9f73d45c4a07d6f424495288a26080 -https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2#d1a88f3ed5b52e1024b80d4bcd26a7a0 -https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.18.1-h7f98852_0.tar.bz2#f26ef8098fab1f719c91eb760d63381a -https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py310h255011f_3.conda#800596144bb613cd7ac58b80900ce835 -https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.1.0-hd9d235c_0.tar.bz2#ebc04a148d7204bb428f8633b89fd3dd -https://conda.anaconda.org/conda-forge/linux-64/cryptography-40.0.2-py310h34c0648_0.conda#991a12eccbca3c9897c62f44b1104a54 -https://conda.anaconda.org/conda-forge/linux-64/curl-8.0.1-h588be90_0.conda#69691e828381dd12df671c26b680f1b0 +https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2023.7.22-hbcca054_0.conda#a73ecd2988327ad4c8f2c331482917f2 +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-h0c91306_1017.conda#3db543896d34fc6804ddfb9239dcb125 +https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.19.1-hd590300_0.conda#e8c18d865be43e2fb3f7a145b6adf1f5 +https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.3.0-hbdc6101_0.conda#797554b8b7603011e8677884381fbcc5 +https://conda.anaconda.org/conda-forge/linux-64/curl-8.2.1-hca28451_0.conda#b7bf35457c5495009392c17feec4fddd https://conda.anaconda.org/conda-forge/linux-64/expat-2.5.0-hcb278e6_1.conda#8b9b5aca60558d02ddaa09d599e55920 -https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.10-nompi_hc118613_107.conda#28b2b46b350ddb6a01d061392f75af54 +https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.10-nompi_hc118613_108.conda#6fa90698000b05dfe8ce6515794fe71a https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.2-h14ed4e7_0.conda#0f69b688f52ff6da70bccb7ff7001d1d https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda#e1232042de76d24539a436d37597eb06 https://conda.anaconda.org/conda-forge/linux-64/freexl-1.0.6-h166bdaf_1.tar.bz2#897e772a157faf3330d72dd291486f62 -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.5.1-py310h8172e47_1.tar.bz2#e3e8495e882625fc0679d321ed383134 -https://conda.anaconda.org/conda-forge/linux-64/geos-3.11.0-h27087fc_0.tar.bz2#a583d0bc9a85c48e8b07a588d1ac8a80 -https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h4fc65e6_3.tar.bz2#dc47fc3ed22615992f89effadd512988 +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.1-py311h815a124_9.conda#e026f17deff5512eeb5119b0e6ba9103 +https://conda.anaconda.org/conda-forge/linux-64/geos-3.12.0-h59595ed_0.conda#3fdf79ef322c8379ae83be491d805369 +https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h22adcc9_11.conda#514167b60f598eaed3f7a60e1dceb9ee https://conda.anaconda.org/conda-forge/linux-64/gettext-0.21.1-h27087fc_0.tar.bz2#14947d8770185e5153fdd04d4673ed37 https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.1-h0b41bf4_3.conda#96f3b11872ef6fad973eac856cd2624f -https://conda.anaconda.org/conda-forge/linux-64/git-2.40.1-pl5321h86e50cf_0.conda#0cb5ff348eb4c201b3b920eff851675d -https://conda.anaconda.org/conda-forge/linux-64/h5py-3.7.0-nompi_py310h06dffec_100.tar.bz2#cc5a27c23e1c2a6fd608846bb09239a8 -https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h9772cbc_5.tar.bz2#ee08782aff2ff9b3291c967fa6bc7336 -https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.1-nompi_h4df4325_104.tar.bz2#ddecfd15c2a412c595c68ff22d3557a0 -https://conda.anaconda.org/conda-forge/linux-64/icu-70.1-h27087fc_0.tar.bz2#87473a15119779e021c314249d4b4aed -https://conda.anaconda.org/conda-forge/linux-64/isce3-0.8.0-py310hc48c5ce_2.tar.bz2#a622fa5f725fc6e2cb762b157ba83ddb -https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h0b41bf4_3.conda#c7a069243e1fbe9a556ed2ec030e6407 -https://conda.anaconda.org/conda-forge/linux-64/json-c-0.16-hc379101_0.tar.bz2#0e2bca6857cb73acec30387fef7c3142 -https://conda.anaconda.org/conda-forge/linux-64/kealib-1.4.15-hfe1a663_0.tar.bz2#32fda93b8db4b6be630d1ee6a94f23b9 +https://conda.anaconda.org/conda-forge/linux-64/git-2.42.0-pl5321h86e50cf_0.conda#96ad24c67e0056d171385859c43218a2 +https://conda.anaconda.org/conda-forge/linux-64/gtest-1.14.0-h00ab1b0_1.conda#d362a81b815334cc921b9362782881f3 +https://conda.anaconda.org/conda-forge/linux-64/h5py-3.9.0-nompi_py311h3839ddf_102.conda#8d9855dc6328f3568740ee1e9414f200 +https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h501b40f_6.conda#c3e9338e15d90106f467377017352b97 +https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.2-nompi_h4f84152_100.conda#2de6a9bc8083b49f09b2f6eb28d3ba3c +https://conda.anaconda.org/conda-forge/linux-64/icu-73.2-h59595ed_0.conda#cc47e1facc155f91abd89b11e48e72ff +https://conda.anaconda.org/conda-forge/linux-64/isce3-0.14.0-py311h360d1b0_1.conda#6d099fec840e5a9c37b71f38b12c27ba +https://conda.anaconda.org/conda-forge/linux-64/json-c-0.17-h7ab15ed_0.conda#9961b1f100c3b6852bd97c9233d06979 +https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.1-hcd42e92_5.conda#d871720bf750347506062ba23a91662d https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 -https://conda.anaconda.org/conda-forge/linux-64/krb5-1.20.1-h81ceb04_0.conda#89a41adce7106749573d883b2f657d78 -https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-h6ed2654_0.tar.bz2#dcc588839de1445d90995a0a2c4f3a39 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.2-h659d440_0.conda#cd95826dbd331ed1be26bdf401432844 +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-haa2dc70_1.conda#980d8aca0bc23ca73fa8caa3e7c84c28 https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda#7aca3059a1729aa76c597603f10b0dd3 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2#76bbff344f0134279f225174e9064c8f -https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-16_linux64_openblas.tar.bz2#d9b7a8639171f6c6fa0a983edabcfe2b -https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_openblas.tar.bz2#20bae26d0a1db73f758fc3754cab4719 -https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.0.1-h588be90_0.conda#b635278a73eb67edcfba7d01a6b48a03 -https://conda.anaconda.org/conda-forge/linux-64/libdap4-3.20.6-hd7c4107_2.tar.bz2#c265ae57e3acdc891f3e2b93cf6784f5 -https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.14-h166bdaf_0.tar.bz2#fc84a0446e4e4fb882e78d786cfb9734 +https://conda.anaconda.org/conda-forge/linux-64/libabseil-20230125.3-cxx17_h59595ed_0.conda#d1db1b8be7c3a8983dcbbbfe4f0765de +https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda#0f683578378cddb223e7fd24f785ab2a +https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.6.2-h039dbb9_1.conda#29cf970521d30d113f3425b84cb250f6 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-18_linux64_openblas.conda#bcddbb497582ece559465b9cd11042e7 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-18_linux64_openblas.conda#93dd9ab275ad888ed8113953769af78c +https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2#c965a5aa0d5c1c37ffc62dff36e28400 +https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.2.1-hca28451_0.conda#96aec6156d58591f5a4e67056521ce1b +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.18-h0b41bf4_0.conda#6aa9c9de5542ecb07fdda9ca626252d8 https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2#4d331e44109e3f0e19b4cb8f9b82f3e1 https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2#6f8720dff19e17ce5d48cfe7f3d2f0a3 https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.5.0-hcb278e6_1.conda#6305a3dd2752c76335295da4e581f2fd https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-12.2.0-h65d4601_19.tar.bz2#e4c94f80aef025c17ab0828cd85ef535 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.5.1-h0c20829_1.tar.bz2#cdf55fbb2010f619d74a9f3e4bc79174 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-12.2.0-h337968e_19.tar.bz2#164b4b1acaedc47ee7e658ae6b308ca3 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-12.2.0-h69a702a_19.tar.bz2#cd7a806282c16e1f2d39a7e80d3a3e0d -https://conda.anaconda.org/conda-forge/linux-64/libglib-2.76.2-hebfc3b9_0.conda#db1d4a1dfc04f3eab50d97551850759a -https://conda.anaconda.org/conda-forge/linux-64/libgomp-12.2.0-h65d4601_19.tar.bz2#cedcee7c064c01c403f962c9e8d3c373 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.1.0-he5830b7_0.conda#cd93f779ff018dd85c7544c015c9db3c +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.1-h880a63b_9.conda#6e41df426ad7c3153554297f57b9017d +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.1.0-h15d22d2_0.conda#afb656a334c409dd9805508af1c89c7a +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.1.0-h69a702a_0.conda#506dc07710dd5b0ba63cbf134897fc10 +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.76.4-hebfc3b9_0.conda#c6f951789c888f7bbd2dd6858eab69de +https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.1.0-he5830b7_0.conda#56ca14d57ac29a75d23a39eb3ee0ddeb +https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-h840a212_1.conda#03c225a73835f5aa68c13e62eb360406 +https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.56.2-h3905398_1.conda#0b01e6ff8002994bd4ddbffcdbec7856 +https://conda.anaconda.org/conda-forge/linux-64/libhwloc-2.9.2-default_h554bfaf_1009.conda#9369f407667517fe52b0e8ed6965ffeb https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2#b62b52da46c39ee2bc3c162ac7f1804d +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-2.1.5.1-h0b41bf4_0.conda#1edd9e67bdb90d78cea97733ff6b54e6 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2#37d3747dd24d604f63d2610910576e63 -https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_openblas.tar.bz2#955d993f41f9354bf753d29864ea20ad -https://conda.anaconda.org/conda-forge/linux-64/libllvm11-11.1.0-he0ac6c6_5.tar.bz2#cae79c6fd61cc6823cbebdbb2c16c60e -https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_h329d8a1_102.tar.bz2#a857af323c5edbd8e9e45fb9facb6f5f +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-18_linux64_openblas.conda#a1244707531e5b143c420c70573c8ec5 +https://conda.anaconda.org/conda-forge/linux-64/libllvm14-14.0.6-hcd5def8_4.conda#73301c133ded2bf71906aa2104edae8b +https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h80fb2b6_112.conda#a19fa6cacf80c8a366572853d5890eb4 https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.52.0-h61bc06f_0.conda#613955a50485812985c059e7b269f42e https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-h7f98852_0.tar.bz2#39b1328babf85c7c3a61636d9cd50206 -https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.21-pthreads_h78a6416_3.tar.bz2#8c5963a49b6035c40646a763293fbb35 +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.24-pthreads_h413a1c8_0.conda#6e4ef6ca28655124dcde9bd500e44c32 https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.39-h753d276_0.conda#e1c890aebdebbfbf87e2c917187b4416 -https://conda.anaconda.org/conda-forge/linux-64/libpq-14.5-hb675445_5.conda#2fcfbd27d237032b4d74144454b75064 -https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-hf730bdb_11.tar.bz2#13b2138ccfbd9821fe13ee0cb7471c03 -https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-hd36657c_19.tar.bz2#d7f5e67dd5cd58da7b78489183e96070 -https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.40.0-h753d276_1.conda#32565f92ca100184c67509d1d91c858a -https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.10.0-hf14f497_3.tar.bz2#d85acad4b47dff4e3def14a769a97906 -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-12.2.0-h46fd767_19.tar.bz2#1030b1f38c129f2634eae026f704fe60 -https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.4.0-h82bc61c_5.conda#e712a63a21f9db647982971dc121cdcf +https://conda.anaconda.org/conda-forge/linux-64/libpq-15.4-hfc447b1_0.conda#b9ce311e7aba8b5fc3122254f0a6e97e +https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.23.3-hd1fb520_1.conda#78c10e8637a6f8d377f9989327d0267d +https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-hb58d41b_14.conda#264f9a3a4ea52c8f4d3e8ae1213a3335 +https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h15f6e67_28.conda#bc9758e23157cb8362e60d3de06aa6fb +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.43.0-h2797004_0.conda#903fa782a9067d5934210df6d79220f6 +https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.0-h0841786_0.conda#1f5a58e686b13bcfde88b93f547d23fe +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.1.0-hfd8a6a1_0.conda#067bcc23164642f4c226da631f2a2e1d +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.1-h8b53f26_1.conda#5b09e13d732dda1a2bc9adc711164f4d https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b -https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.0-h0b41bf4_0.conda#0d4a7508d8c6c65314f2b9c1f56ad408 -https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.13-h7f98852_1004.tar.bz2#b3653fdc58d03face9724f602218a904 -https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.10.3-hca2bb57_4.conda#bb808b654bdc3c783deaf107a2ffb503 -https://conda.anaconda.org/conda-forge/linux-64/libzip-1.9.2-hc929e4a_1.tar.bz2#5b122b50e738c4be5c3f2899f010d7cf -https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-h166bdaf_4.tar.bz2#f3f9de449d32ca9b9c66a22863c96f41 -https://conda.anaconda.org/conda-forge/linux-64/llvmlite-0.39.1-py310h58363a5_1.tar.bz2#18c28e036ae9c5366c56ecd0c875a108 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.1-hd590300_0.conda#82bf6f63eb15ef719b556b63feec3a77 +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.15-h0b41bf4_0.conda#33277193f5b92bad9fdd230eb700929c +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.11.5-h232c23b_1.conda#f3858448893839820d4bcfb14ad3ecdf +https://conda.anaconda.org/conda-forge/linux-64/libzip-1.10.1-h2629f0a_2.conda#a83ad320127e83ae8a86b3db8dfeec77 +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda#f36c115f1ee199da648e0597ec2047ad +https://conda.anaconda.org/conda-forge/linux-64/llvmlite-0.40.1-py311ha6695c7_0.conda#7a2b62d839516ba0cf56717e902229f4 https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda#318b08df404f9c9be5712aaa5a6f0bb0 -https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.3-h27087fc_1.tar.bz2#4acfc691e64342b9dae57cf2adc63238 +https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-h516909a_1000.tar.bz2#bb14fcb13341b81d5eb386423b9d2bac +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-hcb278e6_0.conda#681105bccc2a3f7f1a837d47d39c9179 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda#da0ec11a6454ae19bff5b02ed881a2b1 -https://conda.anaconda.org/conda-forge/linux-64/nss-3.89-he45b914_0.conda#2745719a58eeaab6657256a3f142f099 -https://conda.anaconda.org/conda-forge/linux-64/numba-0.56.4-py310h0e39c9b_1.conda#fe57b83f32db6d332342db71571f501b -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.23.5-py310h53a5b5f_0.conda#3b114b1559def8bad228fec544ac1812 -https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h7d73246_1.tar.bz2#a11b4df9271a8d7917686725aa04c8f2 +https://conda.anaconda.org/conda-forge/linux-64/nss-3.92-h1d7d5a4_0.conda#22c89a3d87828fe925b310b9cdf0f574 +https://conda.anaconda.org/conda-forge/linux-64/numba-0.57.1-py311h96b013e_0.conda#618010d18c4a38073a7f51d9dd3fd8a8 +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.4-py311h64a7726_0.conda#5a03d7c75dd4a9ae9a58850860eca468 +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda#5ce6a42505c6e9e6151c54c3ec8d68ea https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d -https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.0-hd590300_3.conda#8f24d371ed9efb3f0b0de383fb81d51c +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.2-hd590300_0.conda#e5ac5227582d6c83ccf247288c0eb095 https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.40-hc3806b6_0.tar.bz2#69e2c796349cd9b273890bee0febfe1b -https://conda.anaconda.org/conda-forge/linux-64/pcre-8.45-h9c3ff4c_0.tar.bz2#c05d1820a6d34ff07aaaab7a9b7eddaa -https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-2_h7f98852_perl5.tar.bz2#09ba115862623f00962e9809ea248f1a -https://conda.anaconda.org/conda-forge/linux-64/pillow-9.2.0-py310h454ad03_3.tar.bz2#eb354ff791f505b1d6f13f776359d88e +https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-4_hd590300_perl5.conda#3e785bff761095eb7f8676f4694bd1b1 https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2#660e72c82f2e75a6b3fe6a6e75c79f19 -https://conda.anaconda.org/conda-forge/linux-64/poppler-22.04.0-h0733791_3.tar.bz2#74fe2fe238e0cdb31ec61427527a80d7 -https://conda.anaconda.org/conda-forge/linux-64/postgresql-14.5-h3248436_5.conda#eed1ea87210eec1572e7415c1029f04b -https://conda.anaconda.org/conda-forge/linux-64/proj-9.0.1-h93bde94_1.tar.bz2#8259528ea471b0963a91ce174f002e55 +https://conda.anaconda.org/conda-forge/linux-64/poppler-23.08.0-hd18248d_0.conda#59a093146aa911da2ca056c1197e3e41 +https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.4-h8972f4a_0.conda#bf6169ef6f83cc04d8b2a72cd5c364bc +https://conda.anaconda.org/conda-forge/linux-64/proj-9.2.1-ha643af7_0.conda#e992387307f4403ba0ec07d009032550 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2#22dad4df6e8630e8dff2428f6f6a7036 -https://conda.anaconda.org/conda-forge/linux-64/pydantic-1.10.7-py310h1fa729e_0.conda#6306ca76bc0635d84940349cf8d96264 -https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.0-py310hf94497c_0.tar.bz2#d034e57d18be11a94f4922e31539a4a5 -https://conda.anaconda.org/conda-forge/linux-64/pyre-1.11.2-py310hdfe83f4_1.tar.bz2#b681b5319466bb09e4a6dea69d6aea00 -https://conda.anaconda.org/conda-forge/linux-64/python-3.10.10-he550d4f_0_cpython.conda#de25afc7041c103c7f510c746bb63435 -https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.10-3_cp310.conda#4eb33d14d794b0f4be116443ffed3853 -https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py310h5764c6d_5.tar.bz2#9e68d2ff6d98737c855b65f48dd3c597 +https://conda.anaconda.org/conda-forge/linux-64/pydantic-core-2.6.3-py311h46250e7_0.conda#cc8b1e7eab870b5e2a01440d585d2f3d +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.0-py311ha169711_1.conda#92633556d37e88ce45193374d408072c +https://conda.anaconda.org/conda-forge/linux-64/pyre-1.12.1-py311h6b0c3e6_2.conda#917da9c839a1ddba516809a613ed19e2 +https://conda.anaconda.org/conda-forge/linux-64/python-3.11.5-hab00c5b_0_cpython.conda#f0288cb82594b1cbc71111d1cd3c5422 +https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-3_cp311.conda#c2e2630ddb68cf52eec74dc7dfab20b5 +https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py311h459d7ec_0.conda#30eaaf31141e785a445bf1ede6235fe3 +https://conda.anaconda.org/conda-forge/linux-64/re2-2023.03.02-h8c504da_0.conda#206f8fa808748f6e90599c3368a1114e https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4 -https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.17.22-py310h2372a71_0.conda#315a6bdce667bf9673b18c7951da15b6 -https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml.clib-0.2.7-py310h1fa729e_1.conda#2f9b517412af46255cef5e53a22c264e -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.10.1-py310h8deb116_0.conda#4c9604c5ec179c21f8f0a09e3c164480 -https://conda.anaconda.org/conda-forge/linux-64/shapely-1.8.5-py310h5e49deb_1.tar.bz2#e5519576751b59d67164b965a4eb4406 +https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.17.32-py311h459d7ec_0.conda#628868dc17f9bd39a2eb77846e35980c +https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml.clib-0.2.7-py311h2582759_1.conda#5e997292429a22ad50c11af0a2cb0f08 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.2-py311h64a7726_0.conda#18d094fb8e4ac52f93a4f4857a8f1e8f +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.1-py311he06c224_2.conda#10a1953d2f74d292b5de093ceea104b2 https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.10-h9fff704_0.conda#e6d228cd0bb74a51dd18f5bfce0b4115 -https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.40.0-h4ff8645_1.conda#a5cec05bd9e0a5843fa29de9591b93cb -https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.9.5-h3f4058f_0.tar.bz2#4128510b35ed578b909ea68234c9330f +https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.43.0-h2c6b66d_0.conda#713f9eac95d051abe14c3774376854fe +https://conda.anaconda.org/conda-forge/linux-64/tbb-2021.10.0-h00ab1b0_0.conda#9c82b1b389e46b64ec685ec487043e70 +https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.16.3-h84d19f0_1.conda#1cc4e61dc7ca15a570e733a6e20a7b33 https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2#5b8c42eb62e9fc961af70bdd6a26e168 https://conda.anaconda.org/conda-forge/linux-64/tzcode-2023c-h0b41bf4_0.conda#0c0533894f21c3d35697cb8378d390e2 -https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h55805fa_1.tar.bz2#d127dc8efe24033b306180939e51e6af +https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-hac6953d_3.conda#297e6a75dc1b6a440cd341a85eab8a00 https://conda.anaconda.org/conda-forge/linux-64/xorg-kbproto-1.0.7-h7f98852_1002.tar.bz2#4b230e8381279d76131116660f5a241a -https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.0.10-h7f98852_0.tar.bz2#d6b0b50b49eccfe0be0373be628be0f3 -https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.3-hd9c2040_1000.tar.bz2#9e856f78d5c80d5a78f61e72d1d473a3 -https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.4-h0b41bf4_0.conda#ea8fbfeb976ac49cbeb594e985393514 -https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.9-h7f98852_0.tar.bz2#bf6f803a544f26ebbdc3bfff272eb179 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.1-hd590300_0.conda#b462a33c0be1421532f28bfe8f4a7514 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.4-h7391055_0.conda#93ee23f12bc2e684548181256edd2cf6 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.6-h8ee46fc_0.conda#7590b76c3d11d21caa44f3fc38ac584a +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.11-hd590300_0.conda#2c80dc38fface310c9bd81b17037fee5 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.3-h7f98852_0.tar.bz2#be93aabceefa2fac576e971aef407908 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.conda#82b6df12252e6f32402b96dacc656fec -https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2#f59c1242cc1dd93e72c2ee2b360979eb +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.11-hd590300_0.conda#ed67c36f215b310412b2af935bf3e530 https://conda.anaconda.org/conda-forge/linux-64/xorg-renderproto-0.11.1-h7f98852_1002.tar.bz2#06feff3d2634e3097ce2fe681474b534 https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-h0b41bf4_1003.conda#bce9f945da8ad2ae9b1d7165a64d0f87 https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007.tar.bz2#b4a4381d54784606820704f7b5f05a15 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae -https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2#4b11e365c0275b808be78b30f904e295 -https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h3eb15da_6.conda#6b63daed8feeca47be78f323e793d555 +https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-hd590300_5.conda#68c34ec6149623be41a1933ab996a209 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.5-hfc55251_0.conda#04b88013080254850d6c01ed54810589 +https://conda.anaconda.org/conda-forge/noarch/annotated-types-0.5.0-pyhd8ed1ab_0.conda#578ae086f225bc2380c79f3b551ff2f7 https://conda.anaconda.org/conda-forge/noarch/backoff-2.2.1-pyhd8ed1ab_0.tar.bz2#4600709bd85664d8606ae0c76642f8db https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2#9b347a7ec10940d3f7941ff6c460b551 https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2#576d629e47797577ab0f1b351297ef4a -https://conda.anaconda.org/conda-forge/noarch/certifi-2022.12.7-pyhd8ed1ab_0.conda#fb9addc3db06e56abe03e0e9f21a63e6 -https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.1.0-pyhd8ed1ab_0.conda#7fcff9f6f123696e940bda77bd4d6551 +https://conda.anaconda.org/conda-forge/noarch/certifi-2023.7.22-pyhd8ed1ab_0.conda#7f3dbc9179b4dde7da98dfb151d0ad22 +https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.2.0-pyhd8ed1ab_0.conda#313516e9a4b08b12dfb1e1cd390a96e3 https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-hab24e00_0.tar.bz2#19410c3df09dfb12d1206132a1d357c5 -https://conda.anaconda.org/conda-forge/noarch/h5netcdf-1.1.0-pyhd8ed1ab_1.conda#1e64e16b588612c98491cfb09bb24c35 https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2#34272b248891bddccc64479f9a7fffed -https://conda.anaconda.org/conda-forge/noarch/markdown-it-py-2.2.0-pyhd8ed1ab_0.conda#b2928a6c6d52d7e3562b4a59c3214e3a +https://conda.anaconda.org/conda-forge/noarch/markdown-it-py-3.0.0-pyhd8ed1ab_0.conda#93a8e71256479c62074356ef6ebf501b https://conda.anaconda.org/conda-forge/noarch/mdurl-0.1.0-pyhd8ed1ab_0.tar.bz2#f8dab71fdc13b1bf29a01248b156d268 https://conda.anaconda.org/conda-forge/noarch/packaging-23.1-pyhd8ed1ab_0.conda#91cda59e66e1e4afe9476f8ef98f5c30 -https://conda.anaconda.org/conda-forge/noarch/pip-23.1.2-pyhd8ed1ab_0.conda#7288da0d36821349cf1126e8670292df -https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.5.0-pyhd8ed1ab_0.conda#6c36f1c42dd0069b7f23acc74f19be46 +https://conda.anaconda.org/conda-forge/noarch/pip-23.2.1-pyhd8ed1ab_0.conda#e2783aa3f9235225eec92f9081c5b801 +https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.10.0-pyhd8ed1ab_0.conda#0809187ef9b89a3d94a5c24d13936236 https://conda.anaconda.org/conda-forge/noarch/pooch-1.7.0-pyha770c72_3.conda#5936894aade8240c867d292aa0d980c6 https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda#d8d7293c5b37f39b2ac32940621c6592 -https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2#076becd9e05608f8dc72757d5f3a91ff -https://conda.anaconda.org/conda-forge/noarch/pygments-2.15.1-pyhd8ed1ab_0.conda#d316679235612869eba305aa7d41d9bf +https://conda.anaconda.org/conda-forge/noarch/pydantic-2.3.0-pyhd8ed1ab_0.conda#55aaca64695fcebdfa8057c87ed180e7 +https://conda.anaconda.org/conda-forge/noarch/pygments-2.16.1-pyhd8ed1ab_0.conda#40e5cb18165466773619e5c963f00a7b https://conda.anaconda.org/conda-forge/noarch/pymp-pypi-0.4.5-pyhd8ed1ab_0.tar.bz2#3d56b4c5a162223b9f20cef1fdc8a89a -https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.1.1-pyhd8ed1ab_0.conda#0b34aa3ab7e7ccb1765a03dd9ed29938 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2#2a7de29fb590ca14b5243c4c812c8025 -https://conda.anaconda.org/conda-forge/noarch/requests-2.29.0-pyhd8ed1ab_0.conda#5fa992d972fbccfc069161805122cb8d -https://conda.anaconda.org/conda-forge/noarch/rich-13.3.5-pyhd8ed1ab_0.conda#2e40a02ad28e34f26cee2a72042843db -https://conda.anaconda.org/conda-forge/noarch/setuptools-67.7.2-pyhd8ed1ab_0.conda#3b68bc43ec6baa48f7354a446267eefe -https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.1.0-pyh8a188c0_0.tar.bz2#a2995ee828f65687ac5b1e71a2ab1e0c -https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.5.0-hd8ed1ab_0.conda#b3c594fde1a80a1fc3eb9cc4a5dfe392 -https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.5.0-pyha770c72_0.conda#43e7d9e50261fb11deb76e17d8431aac +https://conda.anaconda.org/conda-forge/noarch/requests-2.31.0-pyhd8ed1ab_0.conda#a30144e4156cdbb236f99ebb49828f8b +https://conda.anaconda.org/conda-forge/noarch/rich-13.5.1-pyhd8ed1ab_0.conda#38e7446efa3c8b8a770a0fff862935c0 +https://conda.anaconda.org/conda-forge/noarch/setuptools-68.1.2-pyhd8ed1ab_0.conda#4fe12573bf499ff85a0a364e00cc5c53 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.2.0-pyha21a80b_0.conda#978d03388b62173b8e6f79162cf52b86 +https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.7.1-hd8ed1ab_0.conda#f96688577f1faa58096d06a45136afa2 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.7.1-pyha770c72_0.conda#c39d6a09fe819de4951c2642629d9115 https://conda.anaconda.org/conda-forge/noarch/tzdata-2023c-h71feb2d_0.conda#939e3e74d8be4dac89ce83b20de2492a -https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.15-pyhd8ed1ab_0.conda#27db656619a55d727eaf5a6ece3d2fd6 -https://conda.anaconda.org/conda-forge/noarch/wheel-0.40.0-pyhd8ed1ab_0.conda#49bb0d9e60ce1db25e151780331bb5f3 +https://conda.anaconda.org/conda-forge/noarch/urllib3-2.0.4-pyhd8ed1ab_0.conda#18badd8fa3648d1beb1fcc7f2e0f756e +https://conda.anaconda.org/conda-forge/noarch/wheel-0.41.2-pyhd8ed1ab_0.conda#1ccd092478b3e0ee10d7a891adbf8a4f https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2#cc9f59f147740d88679bf1bd94dbe588 diff --git a/requirements.txt b/requirements.txt index 70e56090..8a8b08cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,7 @@ # These are the packages easily-installable through pip -h5netcdf>=1.0 h5py>=3.6 numba>=0.54 numpy>=1.20 -pillow>=7.0 pydantic>=2.1 pymp-pypi>=0.4.5 pyproj>=3.3 diff --git a/scripts/benchmarking/benchmark-local.sh b/scripts/benchmarking/benchmark-local.sh deleted file mode 100755 index 812b9bec..00000000 --- a/scripts/benchmarking/benchmark-local.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -# Enable common error handling options. -set -o errexit -set -o nounset -set -o pipefail - -USAGE="Usage: $0 -b|--benchmark-dir [-i|--image ]" - -# Parse input arguments -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -i | --image) - IMAGE="$2" - shift - shift - ;; - -b | --benchmark-dir) - BENCH_DIR="$2" - shift - shift - ;; - -h | --help) - echo "$USAGE" - exit 0 - ;; - *) - echo "Unknown option: $key" - exit 1 - ;; - esac -done - -if [ -z "${BENCH_DIR+x}" ]; then - echo "Please specify a benchmark directory." - echo "$USAGE" - exit 1 -fi - -if [ -z "${IMAGE+x}" ]; then - IMAGE="opera-adt/dolphin:latest" - echo "Using default docker image: $IMAGE" -fi - -# Run the SAS workflow. -DATA_DIR="$BENCH_DIR/data" - -# Check that the data directory exists. -if [ ! -d "$DATA_DIR" ]; then - echo "Data directory does not exist: $DATA_DIR" - exit 1 -fi -# Check we have input_slcs/ and dynamic_ancillary/ config_files/ and -# config_files/dolphin_config.yaml -if [ ! -d "$DATA_DIR/input_slcs" ]; then - echo "Input SLCs directory does not exist: $DATA_DIR/input_slcs" - exit 1 -fi -if [ ! -d "$DATA_DIR/dynamic_ancillary" ]; then - echo "Dynamic ancillary directory does not exist: $DATA_DIR/dynamic_ancillary" - exit 1 -fi -if [ ! -d "$DATA_DIR/config_files" ]; then - echo "Config files directory does not exist: $DATA_DIR/config_files" - exit 1 -fi - -cd $DATA_DIR -WORKDIR="/tmp" -mkdir -p scratch output -rm -rf scratch/* output/* -docker run --rm --user=$(id -u):$(id -g) \ - --volume="$(realpath input_slcs):$WORKDIR/input_slcs:ro" \ - --volume="$(realpath dynamic_ancillary):$WORKDIR/dynamic_ancillary:ro" \ - --volume="$(realpath config_files/dolphin_config.yaml):$WORKDIR/dolphin_config.yaml:ro" \ - --volume="$(realpath scratch):$WORKDIR/scratch" \ - --volume="$(realpath output):$WORKDIR/output" \ - --workdir="$WORKDIR" \ - "$IMAGE" dolphin run dolphin_config.yaml - -LOG_DIR="$BENCH_DIR/logs" -mkdir -p $LOG_DIR - -# TODO: run this in a docker container. -eval "$(conda shell.bash hook)" -source ~/anaconda3/etc/profile.d/conda.sh -conda activate mapping -python /u/aurora-r0/staniewi/repos/dolphin/scripts/benchmarking/parse_benchmark_logs.py \ - --config-files config_files/dolphin_config.yaml \ - --outfile "$LOG_DIR/benchmark_results_$(date +%Y%m%d).csv" diff --git a/scripts/benchmarking/parse_benchmark_logs.py b/scripts/benchmarking/parse_benchmark_logs.py deleted file mode 100755 index 601ac9d6..00000000 --- a/scripts/benchmarking/parse_benchmark_logs.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -from __future__ import annotations - -import argparse -import re -import sys -from collections import defaultdict -from datetime import datetime -from pathlib import Path - -import pandas as pd - -from dolphin._types import Filename -from dolphin.workflows._pge_runconfig import RunConfig -from dolphin.workflows.config import Workflow - - -def get_df(dolphin_config_file: Filename): - """Create a dataframe from a directory of log files.""" - if Path(dolphin_config_file).name == "runconfig.yaml": - rc = RunConfig.from_yaml(dolphin_config_file) - alg_file = rc.dynamic_ancillary_file_group.algorithm_parameters_file - if not alg_file.is_absolute(): - alg_file = ( - Path(dolphin_config_file).parent - / rc.dynamic_ancillary_file_group.algorithm_parameters_file - ) - w = rc.to_workflow() - else: - w = Workflow.from_yaml(dolphin_config_file) - - log_file = Path(w.log_file) - - result = _parse_logfile(log_file) - cfg_data = _parse_config(w) - - result.update(cfg_data) - return pd.DataFrame([result]) - - -def _parse_logfile(logfile: Path) -> dict: - out: dict = defaultdict(list) - out["file"] = str(Path(logfile).resolve()) - mempat = r"Maximum memory usage: (\d\.\d{2}) GB" - timepat = ( - r"Total elapsed time for dolphin.workflows.s1_disp.run : (\d*\.\d{2}) minutes" - r" \((\d*\.\d{2}) seconds\)" - ) - wrapped_phase_timepat = ( - r"Total elapsed time for dolphin.workflows.wrapped_phase.run : (\d*\.\d{2})" - r" minutes" - r" \((\d*\.\d{2}) seconds\)" - ) - cfg_version_pat = r"Config file dolphin version: (.*)" - run_version_pat = r"Current running dolphin version: (.*)" - # lines start with datetimes like 2023-04-19 17:13:02 - # so at first line, parse and store the time of execution - for i, line in enumerate(open(logfile).readlines()): - if i == 0: - datetime_str = line[:19] - out["execution_time"] = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S") - if m := re.search(mempat, line): - out["memory"] = float(m.groups()[0]) - continue - if m := re.search(timepat, line): - out["runtime"] = float(m.groups()[1]) - continue - if m := re.search(wrapped_phase_timepat, line): - out["wrapped_phase_runtimes"].append(float(m.groups()[1])) - continue - if m := re.search(cfg_version_pat, line): - out["config_version"] = m.groups()[0] - continue - if m := re.search(run_version_pat, line): - out["run_version"] = m.groups()[0] - continue - return out - - -def _parse_config(workflow: Workflow): - """Grab the relevant parameters from the config file.""" - return { - "block": workflow.worker_settings.block_shape, - "strides": workflow.output_options.strides, - "threads_per_worker": workflow.worker_settings.threads_per_worker, - "n_slc": len(workflow.cslc_file_list), - "n_workers": workflow.worker_settings.n_workers, - "creation_time": workflow.creation_time_utc, - } - - -def _get_cli_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--config-files", - nargs="*", - help="Path(s) to config file", - ) - parser.add_argument( - "-o", - "--outfile", - help="Output file (CSV). If None, outputs as `log_file`.csv", - default=None, - ) - return parser.parse_args() - - -def main(): - """Run main entry point.""" - args = _get_cli_args() - dfs = [] - for config_file in args.config_files: - dfs.append(get_df(config_file)) - - df = pd.concat(dfs) - - if not args.outfile: - # Save as csv file with same directory as log_file - outfile = ( - Path(args.config_files[0]).parent - / f"benchmarks_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv" - ) - else: - outfile = args.outfile - - if not outfile.endswith(".csv"): - outfile = outfile + ".csv" - print(f"Output file must be csv. Writing to {outfile}", file=sys.stderr) - - df.to_csv(outfile, index=False) - - -if __name__ == "__main__": - main() diff --git a/scripts/benchmarking/zenodo.py b/scripts/benchmarking/zenodo.py deleted file mode 100644 index df11dd99..00000000 --- a/scripts/benchmarking/zenodo.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Optional, Union - -import pooch -import requests - -from dolphin._log import get_log -from dolphin._types import Filename - -logger = get_log(__name__) - -RECORD_ID = "1171149" - - -def make_pooch( - record: Union[int, str], sandbox: bool = True, path: Optional[Filename] = None -): - """Create a Pooch instance for a Zenodo record. - - Parameters - ---------- - record : Union[int, str] - The Zenodo record ID. - sandbox : bool, optional - Whether to use the sandbox (default) or the real Zenodo. - path : Optional[Filename], optional - The path to the cache folder. If None (default), use the default - cache folder for the operating system. - - Returns - ------- - pooch.Pooch - The Pooch instance. - """ - base_url, registry = get_zenodo_links(record, sandbox) - dog = pooch.create( - # Use the default cache folder for the operating system - path=path or pooch.os_cache("dolphin"), - base_url=base_url, - # The registry specifies the files that can be fetched - registry=registry, - ) - return dog - - -def get_zenodo_links( - record: Union[int, str], sandbox=True -) -> tuple[str, dict[str, str]]: - """Get the urls and MD5 checksums for the files in a Zenodo record.""" - # Get the record metadata - if sandbox: - url = f"https://sandbox.zenodo.org/api/records/{record}" - else: - url = f"https://zenodo.org/api/records/{record}" - r = requests.get(url) - r.raise_for_status() - data = r.json() - - # Get the files - files = data["files"] - # print the total size of the files - total_size = sum(f["size"] for f in files) - logger.info(f"Total size of {len(files)} files: {total_size / 1e6:.1f} MB") - - # Extract the urls and checksums - urls = [f["links"]["self"] for f in files] - # Get the base url - base_url = "/".join(urls[0].split("/")[:-1]) + "/" - - filenames = [Path(fn).name for fn in urls] - checksums = [f["checksum"] for f in files] - - # Return a dict compatible with the Pooch registry - return base_url, dict(zip(filenames, checksums)) - - -POOCH = make_pooch(RECORD_ID) - - -def get_all_files( - record: Union[int, str], sandbox: bool = True, path: Optional[Filename] = None -): - """Download all the files in a Zenodo record.""" - dog = make_pooch(record, sandbox, path) - # Fetch all the files - for fn in dog.registry_files: - logger.info("Fetching " + fn) - dog.fetch(fn) - return dog diff --git a/scripts/release/generate_product_docx_table.py b/scripts/release/generate_product_docx_table.py deleted file mode 100755 index b17b2812..00000000 --- a/scripts/release/generate_product_docx_table.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -from __future__ import annotations - -import argparse -from itertools import groupby - -import h5py - -from dolphin._types import Filename - - -def generate_docx_table(hdf5_path: Filename, output_path: Filename): - """Create a Word document with a table of HDF5 datasets.""" - # https://python-docx.readthedocs.io/en/latest/user/quickstart.html#adding-a-table - from docx import Document - from docx.enum.table import WD_ROW_HEIGHT_RULE, WD_TABLE_ALIGNMENT - from docx.oxml import parse_xml - from docx.oxml.ns import nsdecls - from docx.shared import Pt - - def _add_row(table, text, height=15, shade=False, bold=False): - # _tc.get_or_add_tcPr().append(shading_elm) - row = table.add_row() - row.height_rule = WD_ROW_HEIGHT_RULE.AT_LEAST - row.height = Pt(height) - if isinstance(text, list): - for i in range(len(text)): - row.cells[i].text = text[i] - else: - row.cells[1].merge(row.cells[0]) - row.cells[1].merge(row.cells[2]) - row.cells[1].text = text - # https://stackoverflow.com/questions/26752856/python-docx-set-table-cell-background-and-text-color # noqa - if shade: - shading_elm = parse_xml(r''.format(nsdecls("w"))) - row.cells[0]._tc.get_or_add_tcPr().append(shading_elm) - # Set the text color to black and remove bold - run = row.cells[0].paragraphs[0].runs[0] - run.font.color.rgb = None - if not bold: - run.font.bold = False - - document = Document() - # Set the default document font to Arial - style = document.styles["Normal"] - font = style.font - font.name = "Arial" - - for group_name, rows in _get_hdf5_attributes_by_group(hdf5_path).items(): - document.add_heading(f"Group: {group_name}", level=2) - table = document.add_table(cols=3, rows=0) - table.style = "Table Grid" # Use the "Table Grid" style to get borders - table.alignment = WD_TABLE_ALIGNMENT.LEFT - - for row in rows: - name = row.pop("Name") - desc = row.pop("Description") - - _add_row(table, f"Name: {name}", shade=True) - - row_text = [f"{k}: {v or 'scalar'}" for k, v in row.items()] - _add_row(table, row_text) - _add_row(table, f"Description: {desc}") - - print(f"Saving to {output_path}") - document.save(output_path) - - -def _get_hdf5_attributes(hdf5_path: Filename) -> list: - table_data = [] - - def append_dset_to_table(name, item): - """Add all dataset's metadata using `visititems`.""" - if not isinstance(item, h5py.Dataset): - return None - data_type = item.dtype - shape = item.shape - description = item.attrs.get("long_name", "") - units = item.attrs.get("units", "") - table_data.append( - dict( - Name=name, - Type=data_type, - Shape=shape, - Units=units, - Description=description, - ) - ) - - with h5py.File(hdf5_path, "r") as hf: - hf.visititems(append_dset_to_table) - return table_data - - -def _get_hdf5_attributes_by_group(hdf5_path: Filename) -> dict[str, list]: - def get_group(name): - try: - return name.split("/")[-2] - except IndexError: - return "root" - - table_data = _get_hdf5_attributes(hdf5_path) - - group_sorted_rows = sorted(table_data, key=lambda row: get_group(row["Name"])) - # Make a dict, where keys are group name, value is the list of rows - # e.g.: { 'DISP': [ {'Name': ,....], 'corrections': [{'Name':...}] - return { - k: list(v) - for k, v in groupby(group_sorted_rows, key=lambda row: get_group(row["Name"])) - } - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("hdf5_path", type=str, help="Path to the HDF5 file") - parser.add_argument( - "output_path", type=str, help="Path to the output Word docx file" - ) - args = parser.parse_args() - generate_docx_table(args.hdf5_path, args.output_path) diff --git a/scripts/release/list_packages.py b/scripts/release/list_packages.py deleted file mode 100755 index 4b76a4fe..00000000 --- a/scripts/release/list_packages.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python -"""list packages installed with conda & yum in CSV format. - -Optionally list packages installed inside a container image instead. -""" -from __future__ import annotations - -import argparse -import csv -import functools -import json -import os -import subprocess -import sys -from dataclasses import astuple, dataclass -from itertools import dropwhile -from pathlib import Path -from typing import Any, Iterator, Optional, TextIO - -DESCRIPTION = __doc__ - - -@dataclass -class Package: - """An installed package.""" - - name: str - version: str - package_manager: str - channel: str - - -class CommandNotFoundError(Exception): - """Raised when a required Unix shell command was not found.""" - - pass - - -class YumListIsAnnoyingError(Exception): - """Raised when 'yum list' does something annoying.""" - - pass - - -def check_command(cmd: str) -> bool: - """Check if a Unix shell command is available.""" - # Check if `cmd` is available on the PATH or as a builtin command. - args = ["command", "-v", cmd] - try: - subprocess.run(args, check=True, stdout=subprocess.DEVNULL) - except subprocess.CalledProcessError as exc: - # The exit status is 0 if the command was found, and 1 if not. - # Any other status is unexpected, so re-raise the error. - if exc.returncode == 1: - return False - else: - raise - else: - return True - - -@functools.lru_cache(maxsize=None) -def conda_command() -> str: - """Get the name of the conda executable.""" - # A list of possible "conda-like" commands to try. - possible_commands = [ - "conda", - "mamba", - "micromamba", - ] - - # Try each possible command. Return the first one that's available. - for cmd in possible_commands: - if check_command(cmd): - return cmd - else: - errmsg = f"conda command not found -- tried {possible_commands}" - raise CommandNotFoundError(errmsg) - - -def list_conda_packages(env: str = "base") -> list[Package]: - """List conda packages installed in the specified environment.""" - # Get the list of conda packages as a JSON-formatted string. - args = [conda_command(), "list", "--name", env, "--json"] - res = subprocess.run(args, capture_output=True, check=True, text=True) - - # Parse the JSON string. - package_list = json.loads(res.stdout) - - def as_package(package_info: dict) -> Package: - name = package_info["name"] - version = package_info["version"] - channel = package_info["channel"] - return Package(name, version, "conda", channel) - - # Convert to `Package` objects. - packages = map(as_package, package_list) - - # Sort alphabetically by name. - return sorted(packages, key=lambda package: package.name) - - -def list_yum_packages() -> list[Package]: - """List system packages installed with yum.""" - # Check if 'yum' is available. - if not check_command("yum"): - errmsg = "'yum' command not found" - raise CommandNotFoundError(errmsg) - - # Get the list of yum packages. - args = ["yum", "list", "installed"] - res = subprocess.run(args, capture_output=True, check=True, text=True) - - # Split the output into a sequence of lines. - # Skip past the first few lines until we get to the actual list of packages. - lines = res.stdout.splitlines() - package_list = dropwhile(lambda line: line != "Installed Packages", lines) - try: - next(package_list) # type: ignore[call-overload] - except StopIteration: - cmd = subprocess.list2cmdline(args) - errmsg = f"unexpected output from {cmd!r}" - raise RuntimeError(errmsg) - - def as_package(line: str) -> Package: - try: - name, version, channel = line.split() - except ValueError as exc: - raise YumListIsAnnoyingError from exc - - # Package names are in '{name}.{arch}' format. - # Strip arch info from package name. - name = ".".join(name.split(".")[:-1]) - - # If the repo cannot be determined, yum prints "installed" by default. - channel = channel.lstrip("@") if (channel != "installed") else "" - - return Package(name, version, "yum", channel) - - def parse_lines(lines: Iterator[str]) -> Iterator[Package]: - for line in lines: - # Sometimes package info in the output is inexplicably broken up - # across two lines... - try: - package = as_package(line) - except YumListIsAnnoyingError: - line += next(lines) - package = as_package(line) - - yield package - - # Parse each line as a `Package` object. - packages = parse_lines(package_list) - - # Sort alphabetically by name. - return sorted(packages, key=lambda package: package.name) - - -def list_packages() -> list[Package]: - """List installed packages.""" - return list_conda_packages() + list_yum_packages() - - -def write_package_csv(csvfile: TextIO = sys.stdout) -> None: - """Write a list of installed packages to file in CSV format.""" - packages = list_packages() - csvwriter = csv.writer(csvfile) - csvwriter.writerow(["Name", "Version", "Package Manager", "Channel"]) - csvwriter.writerows(map(astuple, packages)) - - -def setup_parser() -> argparse.ArgumentParser: - """Set up the command-line argument parser.""" - parser = argparse.ArgumentParser(description=DESCRIPTION) - - docker_group = parser.add_argument_group("docker options") - docker_group.add_argument( - "image", nargs="?", type=str, help="a docker image tag or ID" - ) - docker_group.add_argument( - "-e", - "--entrypoint", - type=str, - help="overwrite the default ENTRYPOINT of the image", - ) - - return parser - - -def parse_args(args: Optional[list[str]] = None) -> dict[str, Any]: - """Parse command-line arguments.""" - parser = setup_parser() - params = parser.parse_args(args) - - if (params.image is None) and (params.entrypoint is not None): - parser.error( - "entrypoint should not be specified unless an image tag or ID is provided" - ) - - return vars(params) - - -def run_in_container(image: str, entrypoint: Optional[str] = None) -> None: - """Run this module inside a container derived from the specified image.""" - # Check if 'docker' is available. - if not check_command("docker"): - errmsg = "'docker' command not found" - raise CommandNotFoundError(errmsg) - - this_file = Path(__file__).resolve() - workdir = Path("/tmp") - - args = [ - "docker", - "run", - "--rm", - f"--user={os.getuid()}:{os.getgid()}", - f"--volume={this_file}:{workdir / 'list_packages.py'}:ro", - f"--workdir={workdir}", - ] - - if entrypoint is not None: - args.append(f"--entrypoint={entrypoint}") - - args += [image, "python", "-m", "list_packages"] - - # Mount this script inside the container and run it without arguments. - subprocess.run(args, check=True) - - -def main(args: Optional[list[str]] = None) -> None: # noqa: D103 - kwargs = parse_args(args) - if kwargs.get("image") is None: - write_package_csv() - else: - run_in_container(**kwargs) - - -if __name__ == "__main__": - main() diff --git a/scripts/release/validate_product.py b/scripts/release/validate_product.py deleted file mode 100755 index 39eab7d2..00000000 --- a/scripts/release/validate_product.py +++ /dev/null @@ -1,520 +0,0 @@ -#!/usr/bin/env python -import argparse -from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional - -import h5py -import numpy as np -from numpy.typing import ArrayLike - -from dolphin import io -from dolphin._log import get_log -from dolphin._types import Filename - -logger = get_log() - -if TYPE_CHECKING: - _SubparserType = argparse._SubParsersAction[argparse.ArgumentParser] -else: - _SubparserType = Any - -DSET_DEFAULT = "unwrapped_phase" - - -class ValidationError(Exception): - """Raised when a product fails a validation check.""" - - pass - - -class ComparisonError(ValidationError): - """Exception raised when two datasets do not match.""" - - pass - - -def compare_groups( - golden_group: h5py.Group, - test_group: h5py.Group, - pixels_failed_threshold: float = 0.01, - diff_threshold: float = 1e-5, -) -> None: - """Compare all datasets in two HDF5 files. - - Parameters - ---------- - golden_group : h5py.Group - Path to the golden file. - test_group : h5py.Group - Path to the test file to be compared. - pixels_failed_threshold : float, optional - The threshold of the percentage of pixels that can fail the comparison. - diff_threshold : float, optional - The abs. difference threshold between pixels to consider failing. - - Raises - ------ - ComparisonError - If the two files do not match in all datasets. - """ - # Check if group names match - if set(golden_group.keys()) != set(test_group.keys()): - raise ComparisonError( - f"Group keys do not match: {set(golden_group.keys())} vs" - f" {set(test_group.keys())}" - ) - - for key in golden_group.keys(): - if isinstance(golden_group[key], h5py.Group): - compare_groups( - golden_group[key], - test_group[key], - pixels_failed_threshold, - diff_threshold, - ) - else: - test_dataset = test_group[key] - golden_dataset = golden_group[key] - _compare_datasets_attr(golden_dataset, test_dataset) - - if key == "connected_component_labels": - _validate_conncomp_labels(test_dataset, golden_dataset) - elif key == "unwrapped_phase": - test_conncomps = test_group["connected_component_labels"] - golden_conncomps = golden_group["connected_component_labels"] - _validate_unwrapped_phase( - test_dataset, - golden_dataset, - test_conncomps, - golden_conncomps, - ) - else: - _validate_dataset( - test_dataset, - golden_dataset, - pixels_failed_threshold, - diff_threshold, - ) - - -def _compare_datasets_attr( - golden_dataset: h5py.Dataset, test_dataset: h5py.Dataset -) -> None: - if golden_dataset.name != test_dataset.name: - raise ComparisonError( - f"Dataset names do not match: {golden_dataset.name} vs {test_dataset.name}" - ) - name = golden_dataset.name - - if golden_dataset.shape != test_dataset.shape: - raise ComparisonError( - f"{name} shapes do not match: {golden_dataset.shape} vs" - f" {test_dataset.shape}" - ) - - if golden_dataset.dtype != test_dataset.dtype: - raise ComparisonError( - f"{name} dtypes do not match: {golden_dataset.dtype} vs" - f" {test_dataset.dtype}" - ) - - if golden_dataset.attrs.keys() != test_dataset.attrs.keys(): - raise ComparisonError( - f"{name} attribute keys do not match: {golden_dataset.attrs.keys()} vs" - f" {test_dataset.attrs.keys()}" - ) - - for attr_key in golden_dataset.attrs.keys(): - if attr_key in ("REFERENCE_LIST", "DIMENSION_LIST"): - continue - val1, val2 = golden_dataset.attrs[attr_key], test_dataset.attrs[attr_key] - if isinstance(val1, np.ndarray): - is_equal = np.allclose(val1, val2, equal_nan=True) - elif isinstance(val1, np.floating) and np.isnan(val1) and np.isnan(val2): - is_equal = True - else: - is_equal = val1 == val2 - if not is_equal: - raise ComparisonError( - f"{name} attribute values for key '{attr_key}' do not match: " - f"{golden_dataset.attrs[attr_key]} vs {test_dataset.attrs[attr_key]}" - ) - - -def _fmt_ratio(num: int, den: int, digits: int = 3) -> str: - """Get a string representation of a rational number as a fraction and percent. - - Parameters - ---------- - num : int - The numerator. - den : int - The denominator. - digits : int, optional - Number of decimal digits to use. Defaults to 3. - - Returns - ------- - str - A string representation of the input. - """ - return f"{num}/{den} ({100.0 * num / den:.{digits}f}%)" - - -def _validate_conncomp_labels( - test_dataset: h5py.Dataset, - ref_dataset: h5py.Dataset, - threshold: float = 0.9, -) -> None: - """Validate connected component labels from unwrapping. - - Computes a binary mask of nonzero-valued labels in the test and reference datasets, - and checks the intersection between the two masks. The dataset fails validation if - the ratio of the intersection area to the reference mask area is below a - predetermined minimum threshold. - - Parameters - ---------- - test_dataset : h5py.Dataset - HDF5 dataset containing connected component labels to be validated. - ref_dataset : h5py.Dataset - HDF5 dataset containing connected component labels to use as reference. Must - have the same shape as `test_dataset`. - threshold : float, optional - Minimum allowable intersection area between nonzero-labeled regions in the test - and reference dataset, as a fraction of the total nonzero-labeled area in the - reference dataset. Must be in the interval [0, 1]. Defaults to 0.9. - - Raises - ------ - ComparisonError - If the intersecting area between the two masks was below the threshold. - """ - logger.info("Checking connected component labels...") - - if test_dataset.shape != ref_dataset.shape: - errmsg = ( - "shape mismatch: test dataset and reference dataset must have the same" - f" shape, got {test_dataset.shape} vs {ref_dataset.shape}" - ) - raise ComparisonError(errmsg) - - if not (0.0 <= threshold <= 1.0): - errmsg = f"threshold must be between 0 and 1, got {threshold}" - raise ValueError(errmsg) - - # Total size of each dataset. - size = ref_dataset.size - - # Compute binary masks of pixels with nonzero labels in each dataset. - test_nonzero = np.not_equal(test_dataset, 0) - ref_nonzero = np.not_equal(ref_dataset, 0) - - # Compute the intersection & union of both masks. - intersect = test_nonzero & ref_nonzero - union = test_nonzero | ref_nonzero - - # Compute the total area of each mask. - test_area = np.sum(test_nonzero) - ref_area = np.sum(ref_nonzero) - intersect_area = np.sum(intersect) - union_area = np.sum(union) - - # Log some statistics about the unwrapped area. - logger.info(f"Test unwrapped area: {_fmt_ratio(test_area, size)}") - logger.info(f"Reference unwrapped area: {_fmt_ratio(ref_area, size)}") - logger.info(f"Intersection/Reference: {_fmt_ratio(intersect_area, ref_area)}") - logger.info(f"Intersection/Union: {_fmt_ratio(intersect_area, union_area)}") - - # Compute the ratio of intersection area to area in the reference mask. - ratio = intersect_area / ref_area - - if ratio < threshold: - errmsg = ( - f"connected component labels dataset {test_dataset.name!r} failed" - " validation: insufficient area of overlap between test and reference" - f" nonzero labels ({ratio} < {threshold})" - ) - raise ComparisonError(errmsg) - - -def _validate_unwrapped_phase( - test_dataset: h5py.Dataset, - ref_dataset: h5py.Dataset, - test_conncomps: ArrayLike, - ref_conncomps: ArrayLike, - nan_threshold: float = 0.01, - atol: float = 1e-6, -) -> None: - """Validate unwrapped phase values against a reference dataset. - - Checks that the phase values in the test dataset are congruent with the reference - dataset -- that is, their values are approximately the same modulo 2pi. - - Parameters - ---------- - test_dataset : h5py.Dataset - HDF5 dataset containing unwrapped phase values to be validated. - ref_dataset : h5py.Dataset - HDF5 dataset containing unwrapped phase values to use as reference. Must have - the same shape as `test_dataset`. - test_conncomps : array_like - Connected component labels associated with `test_dataset`. - ref_conncomps : array_like - Connected component labels associated with `ref_dataset`. - nan_threshold : float - Maximum allowable fraction of NaN values among valid pixels (pixels with nonzero - connected component label). Must be in the interval [0, 1]. Defaults to 0.01. - atol : float, optional - Maximum allowable absolute error between the re-wrapped reference and test - values, in radians. Must be nonnegative. Defaults to 1e-6. - - Raises - ------ - ValidationError - If the NaN value count exceeded the specified threshold. - ComparisonError - If the two datasets were not congruent within the specified error tolerance. - """ - logger.info("Checking unwrapped phase...") - - if test_dataset.shape != ref_dataset.shape: - errmsg = ( - "shape mismatch: test dataset and reference dataset must have the same" - f" shape, got {test_dataset.shape} vs {ref_dataset.shape}" - ) - raise ComparisonError(errmsg) - - if (test_dataset.shape != test_conncomps.shape) or ( - ref_dataset.shape != ref_conncomps.shape - ): - errmsg = ( - "shape mismatch: unwrapped phase and connected component labels must have" - " the same shape" - ) - raise ValidationError(errmsg) - - if not (0.0 <= nan_threshold <= 1.0): - errmsg = f"nan_threshold must be between 0 and 1, got {nan_threshold}" - raise ValueError(errmsg) - - if atol < 0.0: - errmsg = f"atol must be >= 0, got {atol}" - raise ValueError(errmsg) - - # Get a mask of valid pixels (pixels that had nonzero connected component label) in - # both the test & reference data. - test_valid_mask = np.not_equal(test_conncomps, 0) - ref_valid_mask = np.not_equal(ref_conncomps, 0) - valid_mask = test_valid_mask & ref_valid_mask - - # Get the total valid area in both datasets. - test_valid_area = np.sum(test_valid_mask) - ref_valid_area = np.sum(ref_valid_mask) - - # Get a mask of NaN values in either dataset. - test_nan_mask = np.isnan(test_dataset) - ref_nan_mask = np.isnan(ref_dataset) - nan_mask = test_nan_mask | ref_nan_mask - - # Get the total number of NaN values in the valid regions of each dataset. - test_nan_count = np.sum(test_nan_mask & test_valid_mask) - ref_nan_count = np.sum(ref_nan_mask & ref_valid_mask) - - # Log some info about the NaN values. - logger.info(f"Test nan count: {_fmt_ratio(test_nan_count, test_valid_area)}") - logger.info(f"Reference nan count: {_fmt_ratio(ref_nan_count, ref_valid_area)}") - - # Compute the fraction of NaN values in the valid region. - test_nan_frac = test_nan_count / test_valid_area - - if test_nan_frac > nan_threshold: - errmsg = ( - f"unwrapped phase dataset {test_dataset.name!r} failed validation: too" - f" many nan values ({test_nan_frac} > {nan_threshold})" - ) - raise ValidationError(errmsg) - - def rewrap(phi: np.ndarray) -> np.ndarray: - tau = 2.0 * np.pi - return phi - tau * np.ceil((phi - np.pi) / tau) - - # Compute the difference between the test & reference values and wrap it to the - # interval (-pi, pi]. - diff = np.subtract(ref_dataset, test_dataset) - wrapped_diff = rewrap(diff) - - # Mask out invalid pixels and NaN-valued pixels. - wrapped_diff = wrapped_diff[valid_mask & ~nan_mask] - - # Log some statistics about the deviation between the test & reference phase. - abs_wrapped_diff = np.abs(wrapped_diff) - mean_abs_err = np.mean(abs_wrapped_diff) - max_abs_err = np.max(abs_wrapped_diff) - logger.info(f"Mean absolute re-wrapped phase error: {mean_abs_err:.5f} rad") - logger.info(f"Max absolute re-wrapped phase error: {max_abs_err:.5f} rad") - - noncongruent_count = np.sum(abs_wrapped_diff > atol) - logger.info( - "Non-congruent pixel count:" - f" {_fmt_ratio(noncongruent_count, wrapped_diff.size)}" - ) - - if noncongruent_count != 0: - errmsg = ( - f"unwrapped phase dataset {test_dataset.name!r} failed validation: phase" - " values were not congruent with reference dataset" - ) - raise ComparisonError(errmsg) - - -def _validate_dataset( - test_dataset: h5py.Dataset, - golden_dataset: h5py.Dataset, - pixels_failed_threshold: float = 0.01, - diff_threshold: float = 1e-5, -) -> None: - """Validate a generic dataset. - - Parameters - ---------- - test_dataset : h5py.Dataset - HDF5 dataset to be validated. - golden_dataset : h5py.Dataset - HDF5 dataset to use as reference. - pixels_failed_threshold : float, optional - The threshold of the percentage of pixels that can fail the comparison. Defaults - to 0.01. - diff_threshold : float, optional - The abs. difference threshold between pixels to consider failing. Defaults to - 1e-5. - - Raises - ------ - ComparisonError - If the two datasets do not match. - """ - golden = golden_dataset[()] - test = test_dataset[()] - if golden.dtype.kind == "S": - if not np.array_equal(golden, test): - raise ComparisonError(f"Dataset {golden_dataset.name} values do not match") - return - - img_gold = np.ma.masked_invalid(golden) - img_test = np.ma.masked_invalid(test) - abs_diff = np.abs((img_gold.filled(0) - img_test.filled(0))) - num_failed = np.count_nonzero(abs_diff > diff_threshold) - # num_pixels = np.count_nonzero(~np.isnan(img_gold)) # do i want this? - num_pixels = img_gold.size - if num_failed / num_pixels > pixels_failed_threshold: - raise ComparisonError( - f"Dataset {golden_dataset.name} values do not match: Number of" - f" pixels failed: {num_failed} / {num_pixels} =" - f" {100*num_failed / num_pixels:.2f}%" - ) - - -def _check_raster_geometadata(golden_file: Filename, test_file: Filename) -> None: - """Check if the raster metadata (bounds, CRS, and GT) match. - - Parameters - ---------- - golden_file : Filename - Path to the golden file. - test_file : Filename - Path to the test file to be compared. - - Raises - ------ - ComparisonError - If the two files do not match in their metadata - """ - funcs = [io.get_raster_bounds, io.get_raster_crs, io.get_raster_gt] - for func in funcs: - val_golden = func(golden_file) # type: ignore - val_test = func(test_file) # type: ignore - if val_golden != val_test: - raise ComparisonError(f"{func} does not match: {val_golden} vs {val_test}") - - -def _check_compressed_slc_dirs(golden: Filename, test: Filename) -> None: - """Check if the compressed SLC directories match. - - Assumes that the compressed SLC directories are in the same directory as the - `golden` and `test` product files, with the directory name `compressed_slcs`. - - Parameters - ---------- - golden : Filename - Path to the golden file. - test : Filename - Path to the test file to be compared. - - Raises - ------ - ComparisonError - If file names do not match in their compressed SLC directories - """ - golden_slc_dir = Path(golden).parent / "compressed_slcs" - test_slc_dir = Path(test).parent / "compressed_slcs" - - if not golden_slc_dir.exists(): - logger.info("No compressed SLC directory found in golden product.") - return - if not test_slc_dir.exists(): - raise ComparisonError( - f"{test_slc_dir} does not exist, but {golden_slc_dir} exists." - ) - - golden_slc_names = [p.name for p in golden_slc_dir.iterdir()] - test_slc_names = [p.name for p in test_slc_dir.iterdir()] - - if set(golden_slc_names) != set(test_slc_names): - raise ComparisonError( - f"Compressed SLC directories do not match: {golden_slc_names} vs" - f" {test_slc_names}" - ) - - -def compare(golden: Filename, test: Filename, data_dset: str = DSET_DEFAULT) -> None: - """Compare two HDF5 files for consistency.""" - logger.info("Comparing HDF5 contents...") - with h5py.File(golden, "r") as hf_g, h5py.File(test, "r") as hf_t: - compare_groups(hf_g, hf_t) - - logger.info("Checking geospatial metadata...") - _check_raster_geometadata( - io.format_nc_filename(golden, data_dset), - io.format_nc_filename(test, data_dset), - ) - - logger.info(f"Files {golden} and {test} match.") - _check_compressed_slc_dirs(golden, test) - - -def get_parser( - subparser: Optional[_SubparserType] = None, subcommand_name: str = "run" -) -> argparse.ArgumentParser: - """Set up the command line interface.""" - metadata = dict( - description="Compare two HDF5 files for consistency.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - if subparser: - # Used by the subparser to make a nested command line interface - parser = subparser.add_parser(subcommand_name, **metadata) # type: ignore - else: - parser = argparse.ArgumentParser(**metadata) # type: ignore - - parser.add_argument("golden", help="The golden HDF5 file.") - parser.add_argument("test", help="The test HDF5 file to be compared.") - parser.add_argument("--data-dset", default=DSET_DEFAULT) - parser.set_defaults(run_func=compare) - return parser - - -if __name__ == "__main__": - parser = get_parser() - args = parser.parse_args() - compare(args.golden, args.test, args.data_dset) diff --git a/scripts/run_repeated_nrt.py b/scripts/run_repeated_nrt.py deleted file mode 100755 index a329fa13..00000000 --- a/scripts/run_repeated_nrt.py +++ /dev/null @@ -1,455 +0,0 @@ -#!/usr/bin/env python -from __future__ import annotations - -import argparse -from concurrent.futures import ThreadPoolExecutor -from itertools import chain -from pathlib import Path -from typing import Any, Mapping, Sequence - -from dolphin import io, ps, stack, utils -from dolphin._log import get_log, log_runtime -from dolphin._types import Filename -from dolphin.workflows import s1_disp -from dolphin.workflows._utils import group_by_burst, make_nodata_mask -from dolphin.workflows.config import ( - OPERA_DATASET_NAME, - InterferogramNetworkType, - ShpMethod, - Workflow, - WorkflowName, -) - -logger = get_log("dolphin.run_repeated_nrt") - - -def _create_cfg( - *, - slc_files: Sequence[Filename], - half_window_size: tuple[int, int] = (11, 5), - first_ministack: bool = False, - run_unwrap: bool = False, - shp_method: ShpMethod = ShpMethod.GLRT, - amplitude_mean_files: Sequence[Filename] = [], - amplitude_dispersion_files: Sequence[Filename] = [], - strides: Mapping[str, int] = {"x": 6, "y": 3}, - work_dir: Path = Path("."), - n_parallel_bursts: int = 1, -): - # strides = {"x": 1, "y": 1} - interferogram_network: dict[str, Any] - if first_ministack: - interferogram_network = dict( - network_type=InterferogramNetworkType.SINGLE_REFERENCE - ) - workflow_name = WorkflowName.STACK - else: - interferogram_network = dict( - network_type=InterferogramNetworkType.MANUAL_INDEX, - indexes=[(0, -1)], - ) - workflow_name = WorkflowName.SINGLE - - cfg = Workflow( - # Things that change with each workflow run - cslc_file_list=slc_files, - interferogram_network=interferogram_network, - amplitude_mean_files=amplitude_mean_files, - amplitude_dispersion_files=amplitude_dispersion_files, - # Configurable from CLI inputs: - output_options=dict( - strides=strides, - ), - phase_linking=dict( - ministack_size=1000, # for single update, process in one ministack - half_window={"x": half_window_size[0], "y": half_window_size[1]}, - shp_method=shp_method, - ), - scratch_directory=work_dir / "scratch", - output_directory=work_dir / "output", - worker_settings=dict( - # block_size_gb=block_size_gb, - n_parallel_bursts=n_parallel_bursts, - n_workers=4, - threads_per_worker=8, - ), - # ps_options=dict( - # amp_dispersion_threshold=amp_dispersion_threshold, - # ), - # log_file=log_file, - # ) - # Definite hard coded things - unwrap_options=dict( - unwrap_method="snaphu", - run_unwrap=run_unwrap, - # CHANGEME: or else run in background somehow? - ), - save_compressed_slc=True, # always save, and only sometimes will we grab it - workflow_name=workflow_name, - ) - return cfg - - -def get_cli_args() -> argparse.Namespace: - """Set up the command line interface.""" - parser = argparse.ArgumentParser( - description="Repeatedly run the dolphin single update mode.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - # https://docs.python.org/3/library/argparse.html#fromfile-prefix-chars - fromfile_prefix_chars="@", - ) - parser.add_argument( - "--slc-files", - nargs=argparse.ONE_OR_MORE, - help="List the paths of all SLC files to include.", - required=True, - ) - parser.add_argument( - "-ms", - "--ministack-size", - type=int, - default=10, - help="Strides/decimation factor (x, y) (in pixels) to use when determining", - ) - parser.add_argument( - "-hw", - "--half-window-size", - type=int, - nargs=2, - default=(11, 5), - metavar=("X", "Y"), - help="Half window size for the phase linking algorithm", - ) - parser.add_argument( - "--shp-method", - type=ShpMethod, - choices=[s.value for s in ShpMethod], - default=ShpMethod.GLRT, - help="Method used to calculate the SHP.", - ) - parser.add_argument( - "--run-unwrap", - action="store_true", - help="Run the unwrapping stack after phase linking.", - ) - parser.add_argument( - "-j", - "--n-parallel-bursts", - type=int, - default=1, - help="Number of parallel bursts to process.", - ) - parser.add_argument( - "--pre-compute", - action="store_true", - help=( - "Run the amplitude mean/dispersion pre-compute step (not the main" - " workflow)." - ), - ) - return parser.parse_args() - - -@log_runtime -def compute_ps_files( - burst_grouped_slc_files: Mapping[str, Sequence[Filename]], - burst_to_nodata_mask: Mapping[str, Filename], - # max_workers: int = 3, - ps_stack_size: int = 60, - output_folder: Path = Path("precomputed_ps"), -): - """Compute the mean/DA/PS files for each burst group.""" - all_amp_files, all_disp_files, all_ps_files = [], [], [] - # future_burst_dict = {} - # with ThreadPoolExecutor(max_workers=max_workers) as exc: - for burst, file_list in burst_grouped_slc_files.items(): - nodata_mask_file = burst_to_nodata_mask[burst] - # fut = exc.submit( - # _compute_burst_ps_files, - amp_files, disp_files, ps_files = _compute_burst_ps_files( - burst, - file_list, - nodata_mask_file=nodata_mask_file, - ps_stack_size=ps_stack_size, - output_folder=output_folder, - # show_progress=False, - ) - # future_burst_dict[fut] = burst - - # for future in as_completed(future_burst_dict.keys()): - # burst = future_burst_dict[future] - # amp_files, disp_files, ps_files = future.result() - - all_amp_files.extend(amp_files) - all_disp_files.extend(disp_files) - all_ps_files.extend(ps_files) - - logger.info(f"Done with {burst}") - - return all_amp_files, all_disp_files, all_ps_files - - -@log_runtime -def _compute_burst_ps_files( - burst: str, - file_list_all: Sequence[Filename], - nodata_mask_file: Filename, - ps_stack_size: int = 60, - output_folder: Path = Path("precomputed_ps"), -) -> tuple[list[Path], list[Path], list[Path]]: - """Pre-compute the PS files (mean / amp. dispersion) for one burst.""" - logger.info(f"Computing PS files for {burst} into {output_folder}") - vrt_all = stack.VRTStack( - file_list_all, subdataset=OPERA_DATASET_NAME, write_file=False - ) - # logger.info("Created total vrt") - date_list_all = vrt_all.dates - - nodata_mask = io.load_gdal(nodata_mask_file, masked=True).astype(bool).filled(False) - # invert the mask so 1s are the missing data pixels - nodata_mask = ~nodata_mask - - # TODO: fixed number of PS files? fixed time window? - amp_files, disp_files, ps_files = [], [], [] - for full_stack_idx in range(0, len(file_list_all), ps_stack_size): - cur_slice = slice(full_stack_idx, full_stack_idx + ps_stack_size) - cur_files = file_list_all[cur_slice] - cur_dates = date_list_all[cur_slice] - - # Make the current ministack output folder using the start/end dates - d0 = cur_dates[0][0] - d1 = cur_dates[-1][0] - start_end = io._format_date_pair(d0, d1) - basename = f"{burst}_{start_end}" - - # output_folder = output_folder / start_end - output_folder.mkdir(parents=True, exist_ok=True) - cur_vrt = stack.VRTStack( - cur_files, - outfile=output_folder / f"{basename}.vrt", - subdataset=OPERA_DATASET_NAME, - ) - cur_ps_file = (output_folder / f"{basename}_ps_pixels.tif").resolve() - cur_amp_mean = (output_folder / f"{basename}_amp_mean.tif").resolve() - cur_amp_dispersion = ( - output_folder / f"{basename}_amp_dispersion.tif" - ).resolve() - if not all(f.exists() for f in [cur_ps_file, cur_amp_mean, cur_amp_dispersion]): - ps.create_ps( - slc_vrt_file=cur_vrt, - output_amp_mean_file=cur_amp_mean, - output_amp_dispersion_file=cur_amp_dispersion, - output_file=cur_ps_file, - nodata_mask=nodata_mask, - ) - else: - logger.info(f"Skipping existing {basename} files in {output_folder}") - amp_files.append(cur_amp_mean) - disp_files.append(cur_amp_dispersion) - ps_files.append(cur_ps_file) - logger.info(f"Finished with PS processing for {burst}") - return amp_files, disp_files, ps_files - - -def create_nodata_masks( - # date_grouped_slc_files: dict[tuple[datetime.date], list[Filename]], - burst_grouped_slc_files: Mapping[str, Sequence[Filename]], - buffer_pixels: int = 30, - max_workers: int = 3, - output_folder: Path = Path("nodata_masks"), -): - """Create the nodata binary masks for each burst.""" - output_folder.mkdir(exist_ok=True, parents=True) - futures = [] - out_burst_to_file = {} - with ThreadPoolExecutor(max_workers=max_workers) as exc: - for burst, file_list in burst_grouped_slc_files.items(): - outfile = output_folder / f"{burst}.tif" - fut = exc.submit( - make_nodata_mask, - file_list, - outfile, - buffer_pixels=buffer_pixels, - ) - futures.append(fut) - out_burst_to_file[burst] = outfile - for fut in futures: - fut.result() - return out_burst_to_file - - -def _form_burst_vrt_stacks( - burst_grouped_slc_files: Mapping[str, Sequence[Filename]] -) -> dict[str, stack.VRTStack]: - logger.info("For each burst, creating a VRTStack...") - # Each burst needs to be the same size - burst_to_vrt_stack = {} - for b, file_list in burst_grouped_slc_files.items(): - logger.info(f"Checking {len(file_list)} files for {b}") - outfile = Path(f"slc_stack_{b}.vrt") - if not outfile.exists(): - vrt = stack.VRTStack( - file_list, subdataset=OPERA_DATASET_NAME, outfile=outfile - ) - else: - vrt = stack.VRTStack.from_vrt_file(outfile, skip_size_check=True) - burst_to_vrt_stack[b] = vrt - logger.info("Done.") - return burst_to_vrt_stack - - -@log_runtime -def precompute_ps_files(arg_dict) -> None: - """Run the pre-compute step to get means/amp. dispersion for each burst.""" - all_slc_files = arg_dict.pop("slc_files") - - burst_grouped_slc_files = group_by_burst(all_slc_files) - # {'t173_370312_iw2': [PosixPath('t173_370312_iw2_20170203.h5'),... ] } - date_grouped_slc_files = utils.group_by_date(all_slc_files) - # { (datetime.date(2017, 5, 22),) : [PosixPath('t173_370311_iw1_20170522.h5'), ] } - logger.info(f"Found {len(all_slc_files)} total SLC files") - logger.info(f" {len(date_grouped_slc_files)} unique dates,") - logger.info(f" {len(burst_grouped_slc_files)} unique bursts.") - - burst_to_nodata_mask = create_nodata_masks(burst_grouped_slc_files) - - all_amp_files, all_disp_files, all_ps_files = compute_ps_files( - burst_grouped_slc_files, burst_to_nodata_mask - ) - - -def _get_all_slc_files( - burst_to_file_list: Mapping[str, Sequence[Filename]], start_idx: int, end_idx: int -) -> list[Filename]: - return list( - chain.from_iterable( - [file_list[start_idx:end_idx] for file_list in burst_to_file_list.values()] - ) - ) - - -def _run_one_stack( - slc_idx_start: int, - slc_idx_end: int, - ministack_size: int, - burst_to_file_list: Mapping[str, Sequence[Filename]], - comp_slc_files: Sequence[Filename], - all_amp_files: Sequence[Filename], - all_disp_files: Sequence[Filename], -): - cur_path = Path(f"stack_{slc_idx_start}_{slc_idx_end}") - cur_path.mkdir(exist_ok=True) - - logger.info(f"***** START: {cur_path} *****") - # Get the nearest amplitude mean/dispersion files - cur_slc_files = _get_all_slc_files(burst_to_file_list, slc_idx_start, slc_idx_end) - cfg = _create_cfg( - slc_files=list(comp_slc_files) + cur_slc_files, - amplitude_mean_files=all_amp_files, - amplitude_dispersion_files=all_disp_files, - work_dir=cur_path, - **arg_dict, - ) - cfg.to_yaml(cur_path / "dolphin_config.yaml") - s1_disp.run(cfg) - - # On the step before we hit double `ministack_size`, - # archive, shrink, and pull another compressed SLC to replace. - stack_size = slc_idx_end - slc_idx_start - max_stack_size = 2 * ministack_size - 1 # Size at which we archive/shrink - if stack_size == max_stack_size: - # time to shrink! - # Get the compressed SLC that was output - comp_slc_path = (cur_path / "output/compressed_slcs/").resolve() - new_comp_slcs = list(comp_slc_path.glob("*.h5")) - else: - new_comp_slcs = [] - - logger.info(f"***** END: {cur_path} *****") - return new_comp_slcs - - -@log_runtime -def main(arg_dict: dict) -> None: - """Get the command line arguments and run the workflow.""" - arg_dict = vars(args) - ministack_size = arg_dict.pop("ministack_size") - # TODO: verify this is fine to sort them by date? - all_slc_files = sorted(arg_dict.pop("slc_files")) - logger.info(f"Found {len(all_slc_files)} total SLC files") - - # format of `group_by_burst`: - # {'t173_370312_iw2': [PosixPath('t173_370312_iw2_20170203.h5'),... ] } - burst_grouped_slc_files = group_by_burst(all_slc_files) - num_bursts = len(burst_grouped_slc_files) - logger.info(f" {num_bursts} unique bursts.") - # format of `group_by_date`: - # { (datetime.date(2017, 5, 22),) : [PosixPath('t173_370311_iw1_20170522.h5'), ] } - date_grouped_slc_files = utils.group_by_date(all_slc_files) - num_dates = len(date_grouped_slc_files) - logger.info(f" {num_dates} unique dates,") - - burst_to_vrt_stack = _form_burst_vrt_stacks( - burst_grouped_slc_files=burst_grouped_slc_files - ) - burst_to_file_list = {b: v.file_list for b, v in burst_to_vrt_stack.items()} - - # Get the pre-compted PS files (assuming --pre-compute has been run) - all_amp_files = sorted(Path("precomputed_ps/").resolve().glob("*_amp_mean.tif")) - all_disp_files = sorted( - Path("precomputed_ps/").resolve().glob("*_amp_dispersion.tif") - ) - - slc_idx_start = 0 - slc_idx_end = ministack_size - # max_stack_size = 2 * ministack_size - 1 # Size at which we archive/shrink - cur_path = Path(f"stack_{slc_idx_start}_{slc_idx_end}") - cur_path.mkdir(exist_ok=True) - - # TODO: how to make it shift when the year changes for PS files - - # Make the first ministack - cur_slc_files = _get_all_slc_files(burst_to_file_list, slc_idx_start, slc_idx_end) - cfg = _create_cfg( - slc_files=cur_slc_files, - first_ministack=True, - amplitude_mean_files=all_amp_files, - amplitude_dispersion_files=all_disp_files, - work_dir=cur_path, - **arg_dict, - ) - cfg.to_yaml(cur_path / "dolphin_config.yaml") - s1_disp.run(cfg) - - # Rest of mini stacks in incremental-mode - comp_slc_files: list[Path] = [] - slc_idx_end = ministack_size + 1 - - while slc_idx_end <= num_dates: - # we have to wait for the shrink-and-archive jobs before continuing - new_comp_slcs = _run_one_stack( - slc_idx_start, - slc_idx_end, - ministack_size, - burst_to_file_list, - comp_slc_files, - all_amp_files, - all_disp_files, - ) - logger.info( - f"{len(new_comp_slcs)} comp slcs from stack_{slc_idx_start}_{slc_idx_end}" - ) - comp_slc_files.extend(new_comp_slcs) - slc_idx_end += 1 - if len(new_comp_slcs) > 0: - # Move the front idx up by one ministack - slc_idx_start += ministack_size - - -if __name__ == "__main__": - args = get_cli_args() - arg_dict = vars(args) - if arg_dict.pop("pre_compute"): - precompute_ps_files(arg_dict) - else: - main(arg_dict) diff --git a/src/dolphin/py.typed b/src/dolphin/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/src/dolphin/workflows/_cli_run.py b/src/dolphin/workflows/_cli_run.py index c7944980..c271ba12 100644 --- a/src/dolphin/workflows/_cli_run.py +++ b/src/dolphin/workflows/_cli_run.py @@ -11,7 +11,6 @@ def run( config_file: str, debug: bool = False, - pge_format: bool = False, ) -> None: """Run the displacement workflow. @@ -21,24 +20,14 @@ def run( YAML file containing the workflow options. debug : bool, optional Enable debug logging, by default False. - pge_format : bool, optional - If True, the config file is a runconfig in the PGE-expected format. - By default False. """ # rest of imports here so --help doesn't take forever from . import s1_disp - from ._pge_runconfig import RunConfig from .config import Workflow - if pge_format: - pge_rc = RunConfig.from_yaml(config_file) - cfg = pge_rc.to_workflow() - else: - cfg = Workflow.from_yaml(config_file) - pge_rc = None - - s1_disp.run(cfg, debug=debug, pge_runconfig=pge_rc) + cfg = Workflow.from_yaml(config_file) + s1_disp.run(cfg, debug=debug) def get_parser( @@ -64,11 +53,6 @@ def get_parser( action="store_true", help="Print debug messages to the log.", ) - parser.add_argument( - "--pge-format", - action="store_true", - help="Indicate that `config_file` is in the PGE `RunConfig` format.", - ) parser.set_defaults(run_func=run) return parser diff --git a/src/dolphin/workflows/_pge_runconfig.py b/src/dolphin/workflows/_pge_runconfig.py deleted file mode 100644 index 73bd851e..00000000 --- a/src/dolphin/workflows/_pge_runconfig.py +++ /dev/null @@ -1,285 +0,0 @@ -"""Module for creating PGE-compatible run configuration files.""" -from __future__ import annotations - -from pathlib import Path -from typing import ClassVar, List, Optional - -from pydantic import ConfigDict, Field - -from ._yaml_model import YamlModel -from .config import ( - OPERA_DATASET_NAME, - InterferogramNetwork, - OutputOptions, - PhaseLinkingOptions, - PsOptions, - UnwrapOptions, - WorkerSettings, - Workflow, -) - - -class InputFileGroup(YamlModel): - """A group of input files.""" - - cslc_file_list: List[Path] = Field( - default_factory=list, - description="list of paths to CSLC files.", - ) - - frame_id: int = Field( - ..., - description="Frame ID of the bursts contained in `cslc_file_list`.", - ) - model_config = ConfigDict( - extra="forbid", json_schema_extra={"required": ["cslc_file_list", "frame_id"]} - ) - - -class DynamicAncillaryFileGroup(YamlModel, extra="forbid"): - """A group of dynamic ancillary files.""" - - algorithm_parameters_file: Path = Field( # type: ignore - default=..., - description="Path to file containing SAS algorithm parameters.", - ) - amplitude_dispersion_files: List[Path] = Field( - default_factory=list, - description=( - "Paths to existing Amplitude Dispersion files (1 per burst) for PS update" - " calculation. If none provided, computed using the input SLC stack." - ), - ) - amplitude_mean_files: List[Path] = Field( - default_factory=list, - description=( - "Paths to an existing Amplitude Mean files (1 per burst) for PS update" - " calculation. If none provided, computed using the input SLC stack." - ), - ) - geometry_files: List[Path] = Field( - default_factory=list, - description=( - "Paths to the incidence/azimuth-angle files (1 per burst). If none" - " provided, corrections using incidence/azimuth-angle are skipped." - ), - ) - mask_file: Optional[Path] = Field( - None, - description=( - "Optional Byte mask file used to ignore low correlation/bad data (e.g water" - " mask). Convention is 0 for no data/invalid, and 1 for good data. Dtype" - " must be uint8." - ), - ) - dem_file: Optional[Path] = Field( - default=None, - description=( - "Path to the DEM file covering full frame. If none provided, corrections" - " using DEM are skipped." - ), - ) - # TEC file in IONEX format for ionosphere correction - tec_files: Optional[List[Path]] = Field( - default=None, - description=( - "List of paths to TEC files (1 per date) in IONEX format for ionosphere" - " correction. If none provided, ionosphere corrections are skipped." - ), - ) - - # Troposphere weather model - weather_model_files: Optional[List[Path]] = Field( - default=None, - description=( - "List of paths to troposphere weather model files (1 per date). If none" - " provided, troposphere corrections are skipped." - ), - ) - - -class PrimaryExecutable(YamlModel, extra="forbid"): - """Group describing the primary executable.""" - - product_type: str = Field( - default="DISP_S1_SINGLE", - description="Product type of the PGE.", - ) - - -class ProductPathGroup(YamlModel, extra="forbid"): - """Group describing the product paths.""" - - product_path: Path = Field( # type: ignore - default=..., - description="Directory where PGE will place results", - ) - scratch_path: Path = Field( - default=Path("./scratch"), - description="Path to the scratch directory.", - ) - output_directory: Path = Field( - default=Path("./output"), - description="Path to the SAS output directory.", - # The alias means that in the YAML file, the key will be "sas_output_path" - # instead of "output_directory", but the python instance attribute is - # "output_directory" (to match Workflow) - alias="sas_output_path", - ) - product_version: str = Field( - default="0.1", - description="Version of the product, in . format.", - ) - save_compressed_slc: bool = Field( - default=False, - description=( - "Whether the SAS should output and save the Compressed SLCs in addition to" - " the standard product output." - ), - ) - - -class AlgorithmParameters(YamlModel, extra="forbid"): - """Class containing all the other [`Workflow`][dolphin.workflows.config] classes.""" - - # Options for each step in the workflow - ps_options: PsOptions = Field(default_factory=PsOptions) - phase_linking: PhaseLinkingOptions = Field(default_factory=PhaseLinkingOptions) - interferogram_network: InterferogramNetwork = Field( - default_factory=InterferogramNetwork - ) - unwrap_options: UnwrapOptions = Field(default_factory=UnwrapOptions) - output_options: OutputOptions = Field(default_factory=OutputOptions) - subdataset: str = Field( - default=OPERA_DATASET_NAME, - description="Name of the subdataset to use in the input NetCDF files.", - ) - - -class RunConfig(YamlModel, extra="forbid"): - """A PGE run configuration.""" - - # Used for the top-level key - name: ClassVar[str] = "disp_s1_workflow" - - input_file_group: InputFileGroup - dynamic_ancillary_file_group: DynamicAncillaryFileGroup - primary_executable: PrimaryExecutable = Field(default_factory=PrimaryExecutable) - product_path_group: ProductPathGroup - - # General workflow metadata - worker_settings: WorkerSettings = Field(default_factory=WorkerSettings) - - log_file: Optional[Path] = Field( - default=Path("output/disp_s1_workflow.log"), - description="Path to the output log file in addition to logging to stderr.", - ) - - # Override the constructor to allow recursively model_construct without validation - @classmethod - def model_construct(cls, **kwargs): - if "input_file_group" not in kwargs: - kwargs["input_file_group"] = InputFileGroup._construct_empty() - if "dynamic_ancillary_file_group" not in kwargs: - kwargs["dynamic_ancillary_file_group"] = ( - DynamicAncillaryFileGroup._construct_empty() - ) - if "product_path_group" not in kwargs: - kwargs["product_path_group"] = ProductPathGroup._construct_empty() - return super().model_construct( - **kwargs, - ) - - def to_workflow(self): - """Convert to a [`Workflow`][dolphin.workflows.config.Workflow] object.""" - # We need to go to/from the PGE format to our internal Workflow object: - # Note that the top two levels of nesting can be accomplished by wrapping - # the normal model export in a dict. - # - # The things from the RunConfig that are used in the - # Workflow are the input files, PS amp mean/disp files, - # the output directory, and the scratch directory. - # All the other things come from the AlgorithmParameters. - - workflow_name = self.primary_executable.product_type.replace( - "DISP_S1_", "" - ).lower() - cslc_file_list = self.input_file_group.cslc_file_list - output_directory = self.product_path_group.output_directory - scratch_directory = self.product_path_group.scratch_path - mask_file = self.dynamic_ancillary_file_group.mask_file - amplitude_mean_files = self.dynamic_ancillary_file_group.amplitude_mean_files - amplitude_dispersion_files = ( - self.dynamic_ancillary_file_group.amplitude_dispersion_files - ) - - # Load the algorithm parameters from the file - algorithm_parameters = AlgorithmParameters.from_yaml( - self.dynamic_ancillary_file_group.algorithm_parameters_file - ) - param_dict = algorithm_parameters.model_dump() - input_options = dict(subdataset=param_dict.pop("subdataset")) - - # This get's unpacked to load the rest of the parameters for the Workflow - return Workflow( - workflow_name=workflow_name, - cslc_file_list=cslc_file_list, - input_options=input_options, - mask_file=mask_file, - output_directory=output_directory, - scratch_directory=scratch_directory, - save_compressed_slc=self.product_path_group.save_compressed_slc, - amplitude_mean_files=amplitude_mean_files, - amplitude_dispersion_files=amplitude_dispersion_files, - # These ones directly translate - worker_settings=self.worker_settings, - log_file=self.log_file, - # Finally, the rest of the parameters are in the algorithm parameters - **param_dict, - ) - - @classmethod - def from_workflow( - cls, workflow: Workflow, frame_id: int, algorithm_parameters_file: Path - ): - """Convert from a [`Workflow`][dolphin.workflows.config.Workflow] object. - - This is the inverse of the to_workflow method, although there are more - fields in the PGE version, so it's not a 1-1 mapping. - - Since there's no `frame_id` or `algorithm_parameters_file` in the - [`Workflow`][dolphin.workflows.config.Workflow] object, we need to pass - those in as arguments. - - This is mostly used as preliminary setup to further edit the fields. - """ - # Load the algorithm parameters from the file - algo_keys = set(AlgorithmParameters.model_fields.keys()) - alg_param_dict = workflow.model_dump(include=algo_keys) - AlgorithmParameters(**alg_param_dict).to_yaml(algorithm_parameters_file) - # This get's unpacked to load the rest of the parameters for the Workflow - - return cls( - input_file_group=InputFileGroup( - cslc_file_list=workflow.cslc_file_list, - frame_id=frame_id, - ), - dynamic_ancillary_file_group=DynamicAncillaryFileGroup( - algorithm_parameters_file=algorithm_parameters_file, - # amplitude_dispersion_files=workflow.amplitude_dispersion_files, - # amplitude_mean_files=workflow.amplitude_mean_files, - mask_file=workflow.mask_file, - # tec_file=workflow.tec_file, - # weather_model_file=workflow.weather_model_file, - ), - primary_executable=PrimaryExecutable( - product_type=f"DISP_S1_{str(workflow.workflow_name.upper())}", - ), - product_path_group=ProductPathGroup( - product_path=workflow.output_directory, - scratch_path=workflow.scratch_directory, - sas_output_path=workflow.output_directory, - ), - worker_settings=workflow.worker_settings, - log_file=workflow.log_file, - ) diff --git a/src/dolphin/workflows/_product.py b/src/dolphin/workflows/_product.py deleted file mode 100644 index 73cae207..00000000 --- a/src/dolphin/workflows/_product.py +++ /dev/null @@ -1,482 +0,0 @@ -"""Module for creating the OPERA output product in NetCDF format.""" -from __future__ import annotations - -from io import StringIO -from pathlib import Path -from typing import Any, Optional, Sequence, Union - -import h5netcdf -import h5py -import numpy as np -import pyproj -from numpy.typing import ArrayLike, DTypeLike -from PIL import Image - -from dolphin import __version__ as dolphin_version -from dolphin import io -from dolphin._log import get_log -from dolphin._types import Filename -from dolphin.utils import get_dates - -from ._pge_runconfig import RunConfig -from .config import OPERA_DATASET_NAME - -logger = get_log(__name__) - - -CORRECTIONS_GROUP_NAME = "corrections" -IDENTIFICATION_GROUP_NAME = "identification" -GLOBAL_ATTRS = dict( - Conventions="CF-1.8", - contact="operaops@jpl.nasa.gov", - institution="NASA JPL", - mission_name="OPERA", - reference_document="TBD", - title="OPERA L3_DISP_S1 Product", -) - -# Convert chunks to a tuple or h5py errors -HDF5_OPTS = io.DEFAULT_HDF5_OPTIONS.copy() -HDF5_OPTS["chunks"] = tuple(HDF5_OPTS["chunks"]) # type: ignore -# The GRID_MAPPING_DSET variable is used to store the name of the dataset containing -# the grid mapping information, which includes the coordinate reference system (CRS) -# and the GeoTransform. This is in accordance with the CF 1.8 conventions for adding -# geospatial metadata to NetCDF files. -# http://cfconventions.org/cf-conventions/cf-conventions.html#grid-mappings-and-projections -# Note that the name "spatial_ref" used here is arbitrary, but it follows the default -# used by other libraries, such as rioxarray: -# https://github.com/corteva/rioxarray/blob/5783693895b4b055909c5758a72a5d40a365ef11/rioxarray/rioxarray.py#L34 # noqa -GRID_MAPPING_DSET = "spatial_ref" - - -def create_output_product( - unw_filename: Filename, - conncomp_filename: Filename, - tcorr_filename: Filename, - spatial_corr_filename: Filename, - output_name: Filename, - corrections: dict[str, ArrayLike] = {}, - pge_runconfig: Optional[RunConfig] = None, - create_browse_image: bool = True, -): - """Create the OPERA output product in NetCDF format. - - Parameters - ---------- - unw_filename : Filename - The path to the input unwrapped phase image. - conncomp_filename : Filename - The path to the input connected components image. - tcorr_filename : Filename - The path to the input temporal correlation image. - spatial_corr_filename : Filename - The path to the input spatial correlation image. - output_name : Filename, optional - The path to the output NetCDF file, by default "output.nc" - corrections : dict[str, ArrayLike], optional - A dictionary of corrections to write to the output file, by default None - pge_runconfig : Optional[RunConfig], optional - The PGE run configuration, by default None - Used to add extra metadata to the output file. - create_browse_image : bool - If true, creates a PNG browse image of the unwrapped phase - with filename as `output_name`.png - """ - # Read the Geotiff file and its metadata - crs = io.get_raster_crs(unw_filename) - gt = io.get_raster_gt(unw_filename) - unw_arr = io.load_gdal(unw_filename) - - conncomp_arr = io.load_gdal(conncomp_filename) - tcorr_arr = _zero_mantissa(io.load_gdal(tcorr_filename)) - # TODO: add spatial correlation, pass through to function - spatial_corr_arr = _zero_mantissa(io.load_gdal(spatial_corr_filename)) - - # Get the nodata mask (which for snaphu is 0) - mask = unw_arr == 0 - # Set to NaN for final output - unw_arr[mask] = np.nan - - assert unw_arr.shape == conncomp_arr.shape == tcorr_arr.shape - - if create_browse_image: - make_browse_image(Path(output_name).with_suffix(".png"), unw_arr) - - with h5netcdf.File(output_name, "w") as f: - # Create the NetCDF file - f.attrs.update(GLOBAL_ATTRS) - - # Set up the grid mapping variable for each group with rasters - _create_grid_mapping(group=f, crs=crs, gt=gt) - - # Set up the X/Y variables for each group - _create_yx_dsets(group=f, gt=gt, shape=unw_arr.shape) - - # Write the displacement array / conncomp arrays - _create_geo_dataset( - group=f, - name="unwrapped_phase", - data=unw_arr, - description="Unwrapped phase", - fillvalue=np.nan, - attrs=dict(units="radians"), - ) - _create_geo_dataset( - group=f, - name="connected_component_labels", - data=conncomp_arr, - description="Connected component labels of the unwrapped phase", - fillvalue=0, - attrs=dict(units="unitless"), - ) - _create_geo_dataset( - group=f, - name="temporal_correlation", - data=tcorr_arr, - description="Temporal correlation of phase inversion", - fillvalue=np.nan, - attrs=dict(units="unitless"), - ) - _create_geo_dataset( - group=f, - name="spatial_correlation", - data=spatial_corr_arr, - description="Multilooked sample interferometric correlation", - fillvalue=np.nan, - attrs=dict(units="unitless"), - ) - - # Create the group holding phase corrections that were used on the unwrapped phase - corrections_group = f.create_group(CORRECTIONS_GROUP_NAME) - corrections_group.attrs["description"] = ( - "Phase corrections applied to the unwrapped_phase" - ) - - # TODO: Are we going to downsample these for space? - # if so, they need they're own X/Y variables and GeoTransform - _create_grid_mapping(group=corrections_group, crs=crs, gt=gt) - _create_yx_dsets(group=corrections_group, gt=gt, shape=unw_arr.shape) - troposphere = corrections.get("troposphere", np.zeros_like(unw_arr)) - _create_geo_dataset( - group=corrections_group, - name="tropospheric_delay", - data=troposphere, - description="Tropospheric phase delay used to correct the unwrapped phase", - fillvalue=np.nan, - attrs=dict(units="radians"), - ) - ionosphere = corrections.get("ionosphere", np.zeros_like(unw_arr)) - _create_geo_dataset( - group=corrections_group, - name="ionospheric_delay", - data=ionosphere, - description="Ionospheric phase delay used to correct the unwrapped phase", - fillvalue=np.nan, - attrs=dict(units="radians"), - ) - solid_earth = corrections.get("solid_earth", np.zeros_like(unw_arr)) - _create_geo_dataset( - group=corrections_group, - name="solid_earth_tide", - data=solid_earth, - description="Solid Earth tide used to correct the unwrapped phase", - fillvalue=np.nan, - attrs=dict(units="radians"), - ) - plate_motion = corrections.get("plate_motion", np.zeros_like(unw_arr)) - _create_geo_dataset( - group=corrections_group, - name="plate_motion", - data=plate_motion, - description="Phase ramp caused by tectonic plate motion", - fillvalue=np.nan, - attrs=dict(units="radians"), - ) - # Make a scalar dataset for the reference point - reference_point = corrections.get("reference_point", 0.0) - _create_dataset( - group=corrections_group, - name="reference_point", - dimensions=(), - data=reference_point, - fillvalue=0, - description=( - "Dummy dataset containing attributes with the locations where the" - " reference phase was taken." - ), - dtype=int, - # Note: the dataset contains attributes with lists, since the reference - # could have come from multiple points (e.g. some boxcar average of an area). - attrs=dict(units="unitless", rows=[], cols=[], latitudes=[], longitudes=[]), - ) - - # End of the product for non-PGE users - if pge_runconfig is None: - return - - # Add the PGE metadata to the file - with h5netcdf.File(output_name, "a") as f: - identification_group = f.create_group(IDENTIFICATION_GROUP_NAME) - _create_dataset( - group=identification_group, - name="frame_id", - dimensions=(), - data=pge_runconfig.input_file_group.frame_id, - fillvalue=None, - description="ID number of the processed frame.", - attrs=dict(units="unitless"), - ) - # product_version - _create_dataset( - group=identification_group, - name="product_version", - dimensions=(), - data=pge_runconfig.product_path_group.product_version, - fillvalue=None, - description="Version of the product.", - attrs=dict(units="unitless"), - ) - # software_version - _create_dataset( - group=identification_group, - name="software_version", - dimensions=(), - data=dolphin_version, - fillvalue=None, - description="Version of the Dolphin software used to generate the product.", - attrs=dict(units="unitless"), - ) - - # TODO: prob should just make a _to_string method? - ss = StringIO() - pge_runconfig.to_yaml(ss) - runconfig_str = ss.getvalue() - _create_dataset( - group=identification_group, - name="pge_runconfig", - dimensions=(), - data=runconfig_str, - fillvalue=None, - description=( - "The full PGE runconfig YAML file used to generate the product." - ), - attrs=dict(units="unitless"), - ) - - -def _create_dataset( - *, - group: h5netcdf.Group, - name: str, - dimensions: Optional[Sequence[str]], - data: Union[np.ndarray, str], - description: str, - fillvalue: Optional[float], - attrs: Optional[dict[str, Any]] = None, - dtype: Optional[DTypeLike] = None, -) -> h5netcdf.Variable: - if attrs is None: - attrs = {} - attrs.update(long_name=description) - - options = HDF5_OPTS - if isinstance(data, str): - options = {} - # This is a string, so we need to convert it to bytes or it will fail - data = np.string_(data) - elif np.array(data).size <= 1: - # Scalars don't need chunks/compression - options = {} - dset = group.create_variable( - name, - dimensions=dimensions, - data=data, - dtype=dtype, - fillvalue=fillvalue, - **options, - ) - dset.attrs.update(attrs) - return dset - - -def _create_geo_dataset( - *, - group: h5netcdf.Group, - name: str, - data: np.ndarray, - description: str, - fillvalue: float, - attrs: Optional[dict[str, Any]], -) -> h5netcdf.Variable: - dimensions = ["y", "x"] - dset = _create_dataset( - group=group, - name=name, - dimensions=dimensions, - data=data, - description=description, - fillvalue=fillvalue, - attrs=attrs, - ) - dset.attrs["grid_mapping"] = GRID_MAPPING_DSET - return dset - - -def _create_yx_arrays( - gt: list[float], shape: tuple[int, int] -) -> tuple[np.ndarray, np.ndarray]: - """Create the x and y coordinate datasets.""" - ysize, xsize = shape - # Parse the geotransform - x_origin, x_res, _, y_origin, _, y_res = gt - - # Make the x/y arrays - # Note that these are the center of the pixels, whereas the GeoTransform - # is the upper left corner of the top left pixel. - y = np.arange(y_origin + y_res / 2, y_origin + y_res * ysize, y_res) - x = np.arange(x_origin + x_res / 2, x_origin + x_res * xsize, x_res) - return y, x - - -def _create_yx_dsets( - group: h5netcdf.Group, - gt: list[float], - shape: tuple[int, int], -) -> tuple[h5netcdf.Variable, h5netcdf.Variable]: - """Create the x and y coordinate datasets.""" - y, x = _create_yx_arrays(gt, shape) - - if not group.dimensions: - group.dimensions = dict(y=y.size, x=x.size) - # Create the datasets - y_ds = group.create_variable("y", ("y",), data=y, dtype=float) - x_ds = group.create_variable("x", ("x",), data=x, dtype=float) - - for name, ds in zip(["y", "x"], [y_ds, x_ds]): - ds.attrs["standard_name"] = f"projection_{name}_coordinate" - ds.attrs["long_name"] = f"{name.replace('_', ' ')} coordinate of projection" - ds.attrs["units"] = "m" - - return y_ds, x_ds - - -def _create_grid_mapping(group, crs: pyproj.CRS, gt: list[float]) -> h5netcdf.Variable: - """Set up the grid mapping variable.""" - # https://github.com/corteva/rioxarray/blob/21284f67db536d9c104aa872ab0bbc261259e59e/rioxarray/rioxarray.py#L34 - dset = group.create_variable(GRID_MAPPING_DSET, (), data=0, dtype=int) - - dset.attrs.update(crs.to_cf()) - # Also add the GeoTransform - gt_string = " ".join([str(x) for x in gt]) - dset.attrs.update( - dict( - GeoTransform=gt_string, - units="unitless", - long_name=( - "Dummy variable containing geo-referencing metadata in attributes" - ), - ) - ) - - return dset - - -def create_compressed_products(comp_slc_dict: dict[str, Path], output_dir: Filename): - """Make the compressed SLC output product.""" - - def form_name(filename: Path, burst: str): - # filename: compressed_20180222_20180716.tif - date_str = io._format_date_pair(*get_dates(filename.stem)) - return f"compressed_slc_{burst}_{date_str}.h5" - - attrs = GLOBAL_ATTRS.copy() - attrs["title"] = "Compressed SLC" - *parts, dset_name = OPERA_DATASET_NAME.split("/") - group_name = "/".join(parts) - - for burst, comp_slc_file in comp_slc_dict.items(): - outname = Path(output_dir) / form_name(comp_slc_file, burst) - if outname.exists(): - logger.info(f"Skipping existing {outname}") - continue - - crs = io.get_raster_crs(comp_slc_file) - gt = io.get_raster_gt(comp_slc_file) - data = _zero_mantissa(io.load_gdal(comp_slc_file)) - - logger.info(f"Writing {outname}") - with h5py.File(outname, "w") as hf: - # add type to root for GDAL recognition of complex datasets in NetCDF - ctype = h5py.h5t.py_create(np.complex64) - ctype.commit(hf["/"].id, np.string_("complex64")) - - with h5netcdf.File(outname, mode="a", invalid_netcdf=True) as f: - f.attrs.update(attrs) - - data_group = f.create_group(group_name) - _create_grid_mapping(group=data_group, crs=crs, gt=gt) - _create_yx_dsets(group=data_group, gt=gt, shape=data.shape) - _create_geo_dataset( - group=data_group, - name=dset_name, - data=data, - description="Compressed SLC product", - fillvalue=np.nan + 0j, - attrs=dict(units="unitless"), - ) - - -def _zero_mantissa(data: np.ndarray, bits_to_keep: int = 10): - """Zero out 23-`bits_to_keep` bits of the mantissa of a float32 array. - - This is used to make the data more compressible when we don't need the - full precision (e.g. for correlation estimates). - - By default, this will zero out 13 bits, which (for data between 0 and 1) - is `1 / 2**13 ~= 0.0001` of precision. - """ - float32_mantissa_bits = 23 - nzero = float32_mantissa_bits - bits_to_keep - - # Start with 0b11111111111111111111111111111111 - allbits = (1 << 32) - 1 - # Shift it to the left by `nzero` bits - bitmask = (allbits << nzero) & allbits - # Mask out the least significant `nzero` bits - if np.iscomplexobj(data): - dr = data.real.view(np.uint32) - dr &= bitmask - di = data.imag.view(np.uint32) - di &= bitmask - else: - dr = data.view(np.uint32) - dr &= bitmask - return data - - -def make_browse_image( - output_filename: Filename, - arr: ArrayLike, - max_dim_allowed: int = 2048, -) -> None: - """Create a PNG browse image for the output product. - - Parameters - ---------- - output_filename : Filename - Name of output PNG - arr : ArrayLike - input 2D image array - max_dim_allowed : int, default = 2048 - Size (in pixels) of the maximum allowed dimension of output image. - Image gets rescaled with same aspect ratio. - """ - orig_shape = arr.shape - scaling_ratio = max([s / max_dim_allowed for s in orig_shape]) - # scale original shape by scaling ratio - scaled_shape = [int(np.ceil(s / scaling_ratio)) for s in orig_shape] - - # TODO: Make actual browse image - dummy = np.zeros(scaled_shape, dtype="uint8") - img = Image.fromarray(dummy, mode="L") - img.save(output_filename, transparency=0) diff --git a/src/dolphin/workflows/_utils.py b/src/dolphin/workflows/_utils.py index 7d2df5ef..cc91aa71 100644 --- a/src/dolphin/workflows/_utils.py +++ b/src/dolphin/workflows/_utils.py @@ -15,7 +15,7 @@ from dolphin._log import get_log from dolphin._types import Filename -from .config import OPERA_BURST_RE, OPERA_DATASET_NAME, OPERA_IDENTIFICATION +from .config import OPERA_BURST_RE, OPERA_DATASET_NAME, OPERA_IDENTIFICATION, Workflow logger = get_log(__name__) @@ -207,3 +207,28 @@ def make_nodata_mask( cmd = f"gdal_rasterize -q -burn 1 {temp_vector_file} {out_file}" logger.info(cmd) subprocess.check_call(cmd, shell=True) + + +def _create_burst_cfg( + cfg: Workflow, + burst_id: str, + grouped_slc_files: dict[str, list[Path]], + grouped_amp_mean_files: dict[str, list[Path]], + grouped_amp_dispersion_files: dict[str, list[Path]], +) -> Workflow: + cfg_temp_dict = cfg.model_dump(exclude={"cslc_file_list"}) + + # Just update the inputs and the work directory + top_level_work = cfg_temp_dict["work_directory"] + cfg_temp_dict.update({"work_directory": top_level_work / burst_id}) + cfg_temp_dict["cslc_file_list"] = grouped_slc_files[burst_id] + cfg_temp_dict["amplitude_mean_files"] = grouped_amp_mean_files[burst_id] + cfg_temp_dict["amplitude_dispersion_files"] = grouped_amp_dispersion_files[burst_id] + return Workflow(**cfg_temp_dict) + + +def _remove_dir_if_empty(d: Path) -> None: + try: + d.rmdir() + except OSError: + pass diff --git a/src/dolphin/workflows/config.py b/src/dolphin/workflows/config.py index e8e742a7..a79d0f04 100644 --- a/src/dolphin/workflows/config.py +++ b/src/dolphin/workflows/config.py @@ -326,14 +326,9 @@ class Workflow(YamlModel): " uint8." ), ) - scratch_directory: Path = Field( - Path("scratch"), - description="Name of sub-directory to use for scratch files", - validate_default=True, - ) - output_directory: Path = Field( - Path("output"), - description="Name of sub-directory to use for output files", + work_directory: Path = Field( + Path("."), + description="Name of sub-directory to use for writing output files", validate_default=True, ) @@ -387,7 +382,7 @@ class Workflow(YamlModel): extra="forbid", json_schema_extra={"required": ["cslc_file_list"]} ) - @field_validator("output_directory", "scratch_directory") + @field_validator("work_directory") @classmethod def _make_dir_absolute(cls, v: Path): return v.resolve() @@ -467,13 +462,13 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: """After validation, set up properties for use during workflow run.""" super().__init__(*args, **kwargs) - # Ensure outputs from workflow steps are within scratch directory. - scratch_dir = self.scratch_directory + # Ensure outputs from workflow steps are within work directory. + work_dir = self.work_directory # Save all directories as absolute paths - scratch_dir = scratch_dir.resolve(strict=False) + work_dir = work_dir.resolve(strict=False) # For each workflow step that has an output folder, move it inside - # the scratch directory (if it's not already inside). + # the work directory (if it's not already inside). # They may already be inside if we're loading from a json/yaml file. for step in [ "ps_options", @@ -482,25 +477,24 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "unwrap_options", ]: opts = getattr(self, step) - if not opts._directory.parent == scratch_dir: - opts._directory = scratch_dir / opts._directory + if not opts._directory.parent == work_dir: + opts._directory = work_dir / opts._directory opts._directory = opts._directory.resolve(strict=False) # Track the directories that need to be created at start of workflow self._directory_list = [ - scratch_dir, - self.output_directory, + work_dir, self.ps_options._directory, self.phase_linking._directory, self.interferogram_network._directory, self.unwrap_options._directory, ] # Add the output PS files we'll create to the `PS` directory, making - # sure they're inside the scratch directory + # sure they're inside the work directory ps_opts = self.ps_options - ps_opts._amp_dispersion_file = scratch_dir / ps_opts._amp_dispersion_file - ps_opts._amp_mean_file = scratch_dir / ps_opts._amp_mean_file - ps_opts._output_file = scratch_dir / ps_opts._output_file + ps_opts._amp_dispersion_file = work_dir / ps_opts._amp_dispersion_file + ps_opts._amp_mean_file = work_dir / ps_opts._amp_mean_file + ps_opts._output_file = work_dir / ps_opts._output_file def create_dir_tree(self, debug=False): """Create the directory tree for the workflow.""" diff --git a/src/dolphin/workflows/s1_disp.py b/src/dolphin/workflows/s1_disp.py index a7e37942..0d11b559 100755 --- a/src/dolphin/workflows/s1_disp.py +++ b/src/dolphin/workflows/s1_disp.py @@ -5,16 +5,14 @@ from concurrent.futures import ProcessPoolExecutor from pathlib import Path from pprint import pformat -from typing import Optional from dolphin import __version__ from dolphin._background import DummyProcessPoolExecutor from dolphin._log import get_log, log_runtime from dolphin.utils import get_max_memory_usage, set_num_threads -from . import _product, stitch_and_unwrap, wrapped_phase -from ._pge_runconfig import RunConfig -from ._utils import group_by_burst +from . import stitch_and_unwrap, wrapped_phase +from ._utils import _create_burst_cfg, _remove_dir_if_empty, group_by_burst from .config import Workflow @@ -22,7 +20,6 @@ def run( cfg: Workflow, debug: bool = False, - pge_runconfig: Optional[RunConfig] = None, ): """Run the displacement workflow on a stack of SLCs. @@ -33,9 +30,6 @@ def run( workflow. debug : bool, optional Enable debug logging, by default False. - pge_runconfig : RunConfig, optional - If provided, adds PGE-specific metadata to the output product. - Not used by the workflow itself, only for extra metadata. """ # Set the logging level for all `dolphin.` modules logger = get_log(name="dolphin", debug=debug, filename=cfg.log_file) @@ -139,62 +133,8 @@ def run( ) ) - # ###################################### - # 3. Finalize the output as an HDF5 product - # ###################################### - logger.info(f"Creating {len(unwrapped_paths)} outputs in {cfg.output_directory}") - for unw_p, cc_p, s_corr_p in zip( - unwrapped_paths, - conncomp_paths, - spatial_corr_paths, - ): - output_name = cfg.output_directory / unw_p.with_suffix(".nc").name - _product.create_output_product( - unw_filename=unw_p, - conncomp_filename=cc_p, - tcorr_filename=stitched_tcorr_file, - spatial_corr_filename=s_corr_p, - output_name=output_name, - corrections={}, - pge_runconfig=pge_runconfig, - ) - - if cfg.save_compressed_slc: - logger.info(f"Saving {len(comp_slc_dict.items())} compressed SLCs") - output_dir = cfg.output_directory / "compressed_slcs" - output_dir.mkdir(exist_ok=True) - _product.create_compressed_products( - comp_slc_dict=comp_slc_dict, - output_dir=output_dir, - ) - # Print the maximum memory usage for each worker max_mem = get_max_memory_usage(units="GB") logger.info(f"Maximum memory usage: {max_mem:.2f} GB") logger.info(f"Config file dolphin version: {cfg._dolphin_version}") logger.info(f"Current running dolphin version: {__version__}") - - -def _create_burst_cfg( - cfg: Workflow, - burst_id: str, - grouped_slc_files: dict[str, list[Path]], - grouped_amp_mean_files: dict[str, list[Path]], - grouped_amp_dispersion_files: dict[str, list[Path]], -) -> Workflow: - cfg_temp_dict = cfg.model_dump(exclude={"cslc_file_list"}) - - # Just update the inputs and the scratch directory - top_level_scratch = cfg_temp_dict["scratch_directory"] - cfg_temp_dict.update({"scratch_directory": top_level_scratch / burst_id}) - cfg_temp_dict["cslc_file_list"] = grouped_slc_files[burst_id] - cfg_temp_dict["amplitude_mean_files"] = grouped_amp_mean_files[burst_id] - cfg_temp_dict["amplitude_dispersion_files"] = grouped_amp_dispersion_files[burst_id] - return Workflow(**cfg_temp_dict) - - -def _remove_dir_if_empty(d: Path) -> None: - try: - d.rmdir() - except OSError: - pass diff --git a/src/dolphin/workflows/stitch_and_unwrap.py b/src/dolphin/workflows/stitch_and_unwrap.py index c00c11e1..98b06ff3 100644 --- a/src/dolphin/workflows/stitch_and_unwrap.py +++ b/src/dolphin/workflows/stitch_and_unwrap.py @@ -4,10 +4,20 @@ from pathlib import Path from typing import Sequence -from dolphin import io, stitching, unwrap +from dolphin import io, stitching from dolphin._log import get_log, log_runtime from dolphin.interferogram import estimate_correlation_from_phase +try: + from dolphin import unwrap + + UNWRAP_INSTALLED = False +except ImportError as e: + logger = get_log(__name__) + logger.info("Unwrapping dependencies not installed: %s", e) + UNWRAP_INSTALLED = False + + from .config import UnwrapMethod, Workflow @@ -132,24 +142,23 @@ def run( ifg_filenames = sorted(Path(stitched_ifg_dir).glob("*.int")) # type: ignore if not ifg_filenames: raise FileNotFoundError(f"No interferograms found in {stitched_ifg_dir}") - unwrapped_paths, conncomp_paths = unwrap.run( - ifg_filenames=ifg_filenames, - cor_filenames=spatial_corr_paths, - output_path=cfg.unwrap_options._directory, - nlooks=nlooks, - mask_file=output_mask, - # mask_file: Optional[Filename] = None, - # TODO: max jobs based on the CPUs and the available RAM? use dask? - max_jobs=unwrap_jobs, - # overwrite: bool = False, - no_tile=True, - use_icu=use_icu, - ) - - # #################### - # 3. Phase Corrections - # #################### - # TODO: Determine format for the tropospheric/ionospheric phase correction + if UNWRAP_INSTALLED: + unwrapped_paths, conncomp_paths = unwrap.run( + ifg_filenames=ifg_filenames, + cor_filenames=spatial_corr_paths, + output_path=cfg.unwrap_options._directory, + nlooks=nlooks, + mask_file=output_mask, + # mask_file: Optional[Filename] = None, + # TODO: max jobs based on the CPUs and the available RAM? use dask? + max_jobs=unwrap_jobs, + # overwrite: bool = False, + no_tile=True, + use_icu=use_icu, + ) + else: + logger.info("Unwrapping dependencies not installed, skipping.") + unwrapped_paths, conncomp_paths = [], [] return unwrapped_paths, conncomp_paths, spatial_corr_paths, stitched_tcorr_file diff --git a/src/dolphin/workflows/wrapped_phase.py b/src/dolphin/workflows/wrapped_phase.py index 28979658..cdcbeaf4 100644 --- a/src/dolphin/workflows/wrapped_phase.py +++ b/src/dolphin/workflows/wrapped_phase.py @@ -35,7 +35,7 @@ def run(cfg: Workflow, debug: bool = False) -> tuple[list[Path], Path, Path, Pat In the case of sequential phase linking, this is the average tcorr file. """ logger = get_log(debug=debug) - logger.info(f"Running wrapped phase estimation in {cfg.scratch_directory}") + logger.info(f"Running wrapped phase estimation in {cfg.work_directory}") input_file_list = cfg.cslc_file_list if not input_file_list: @@ -48,12 +48,12 @@ def run(cfg: Workflow, debug: bool = False) -> tuple[list[Path], Path, Path, Pat vrt_stack = stack.VRTStack( input_file_list, subdataset=subdataset, - outfile=cfg.scratch_directory / "slc_stack.vrt", + outfile=cfg.work_directory / "slc_stack.vrt", ) # Make the nodata mask from the polygons, if we're using OPERA CSLCs try: - nodata_mask_file = cfg.scratch_directory / "nodata_mask.tif" + nodata_mask_file = cfg.work_directory / "nodata_mask.tif" _utils.make_nodata_mask( vrt_stack.file_list, out_file=nodata_mask_file, buffer_pixels=200 ) diff --git a/tests/test_workflows_config.py b/tests/test_workflows_config.py index 83abfdde..e450e85b 100644 --- a/tests/test_workflows_config.py +++ b/tests/test_workflows_config.py @@ -280,22 +280,19 @@ def test_config_defaults(dir_with_1_slc): assert c.output_options == config.OutputOptions() assert c.worker_settings == config.WorkerSettings() assert c.input_options == config.InputOptions(subdataset="data") - assert c.output_directory == Path("output").resolve() - assert c.scratch_directory == Path("scratch").resolve() + assert c.work_directory == Path(".").resolve() # Check the defaults for the sub-configs, where the folders - # should have been moved to the scratch directory - assert c.ps_options._directory == Path("scratch/PS").resolve() - assert c.ps_options._amp_mean_file == Path("scratch/PS/amp_mean.tif").resolve() + # should have been moved to the working directory + assert c.ps_options._directory == Path("PS").resolve() + assert c.ps_options._amp_mean_file == Path("PS/amp_mean.tif").resolve() - p = Path("scratch/PS/amp_dispersion.tif") + p = Path("PS/amp_dispersion.tif") assert c.ps_options._amp_dispersion_file == p.resolve() - assert c.phase_linking._directory == Path("scratch/linked_phase").resolve() + assert c.phase_linking._directory == Path("linked_phase").resolve() - assert ( - c.interferogram_network._directory == Path("scratch/interferograms").resolve() - ) + assert c.interferogram_network._directory == Path("interferograms").resolve() assert c.interferogram_network.reference_idx == 0 assert ( c.interferogram_network.network_type @@ -305,7 +302,7 @@ def test_config_defaults(dir_with_1_slc): assert c.interferogram_network.max_bandwidth is None assert c.interferogram_network.max_temporal_baseline is None - assert c.unwrap_options._directory == Path("scratch/unwrapped").resolve() + assert c.unwrap_options._directory == Path("unwrapped").resolve() now = datetime.utcnow() assert (now - c.creation_time_utc).seconds == 0 @@ -325,8 +322,8 @@ def test_config_create_dir_tree(tmpdir, slc_file_list_nc): assert c.phase_linking._directory.exists() assert c.unwrap_options._directory.exists() - # Check that the scratch directory is created - assert Path("scratch").exists() + # Check that the working directory is created + assert Path(".").exists() for d in c._directory_list: assert d.exists() diff --git a/tests/test_workflows_pge_runconfig.py b/tests/test_workflows_pge_runconfig.py deleted file mode 100644 index 9a6b62b7..00000000 --- a/tests/test_workflows_pge_runconfig.py +++ /dev/null @@ -1,92 +0,0 @@ -import sys -import warnings - -import pytest - -from dolphin.workflows._pge_runconfig import ( - AlgorithmParameters, - DynamicAncillaryFileGroup, - InputFileGroup, - PrimaryExecutable, - ProductPathGroup, - RunConfig, -) - - -def test_algorithm_parameters_schema(): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=UserWarning) - AlgorithmParameters.print_yaml_schema() - - -def test_run_config_schema(): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=UserWarning) - RunConfig.print_yaml_schema() - - -@pytest.fixture -def input_file_group(slc_file_list_nc_with_sds): - return InputFileGroup(cslc_file_list=slc_file_list_nc_with_sds, frame_id=10) - - -@pytest.fixture -def algorithm_parameters_file(tmp_path): - f = tmp_path / "test.yaml" - AlgorithmParameters().to_yaml(f) - return f - - -@pytest.fixture -def dynamic_ancillary_file_group(algorithm_parameters_file): - return DynamicAncillaryFileGroup( - algorithm_parameters_file=algorithm_parameters_file - ) - - -@pytest.fixture -def product_path_group(tmp_path): - product_path = tmp_path / "product_path" - product_path.mkdir() - return ProductPathGroup(product_path=product_path) - - -@pytest.fixture -def runconfig_minimum( - input_file_group, - dynamic_ancillary_file_group, - product_path_group, -): - c = RunConfig( - input_file_group=input_file_group, - primary_executable=PrimaryExecutable(), - dynamic_ancillary_file_group=dynamic_ancillary_file_group, - product_path_group=product_path_group, - ) - return c - - -def test_runconfig_to_yaml(runconfig_minimum): - print(runconfig_minimum.to_yaml(sys.stdout)) - - -def test_runconfig_to_workflow(runconfig_minimum): - print(runconfig_minimum.to_workflow()) - - -def test_runconfig_from_workflow(tmp_path, runconfig_minimum): - w = runconfig_minimum.to_workflow() - frame_id = runconfig_minimum.input_file_group.frame_id - algo_file = tmp_path / "algo_params.yaml" - w2 = RunConfig.from_workflow(w, frame_id, algo_file).to_workflow() - - # these will be slightly different - w2.creation_time_utc = w.creation_time_utc - assert w == w2 - - -def test_runconfig_yaml_rountrip(tmp_path, runconfig_minimum): - f = tmp_path / "test.yaml" - runconfig_minimum.to_yaml(f) - c = RunConfig.from_yaml(f) - assert c == runconfig_minimum diff --git a/tests/test_workflows_product.py b/tests/test_workflows_product.py deleted file mode 100644 index f42461c3..00000000 --- a/tests/test_workflows_product.py +++ /dev/null @@ -1,69 +0,0 @@ -import numpy as np -import pytest - -from dolphin import io -from dolphin.workflows._product import create_output_product - -# random place in hawaii -GEOTRANSFORM = [204500.0, 5.0, 0.0, 2151300.0, 0.0, -10.0] -SRS = "EPSG:32605" -SHAPE = (256, 256) - - -@pytest.fixture -def unw_filename(tmp_path) -> str: - data = np.random.randn(*SHAPE).astype(np.float32) - filename = tmp_path / "unw.tif" - io.write_arr( - arr=data, output_name=filename, geotransform=GEOTRANSFORM, projection=SRS - ) - return filename - - -@pytest.fixture -def conncomp_filename(tmp_path) -> str: - data = np.random.randn(*SHAPE).astype(np.uint32) - filename = tmp_path / "conncomp.tif" - io.write_arr( - arr=data, output_name=filename, geotransform=GEOTRANSFORM, projection=SRS - ) - return filename - - -@pytest.fixture -def tcorr_filename(tmp_path) -> str: - data = np.random.randn(*SHAPE).astype(np.float32) - filename = tmp_path / "tcorr.tif" - io.write_arr( - arr=data, output_name=filename, geotransform=GEOTRANSFORM, projection=SRS - ) - return filename - - -@pytest.fixture -def spatial_corr_filename(tmp_path) -> str: - data = np.random.randn(*SHAPE).astype(np.float32) - filename = tmp_path / "spatial_corr.tif" - io.write_arr( - arr=data, output_name=filename, geotransform=GEOTRANSFORM, projection=SRS - ) - return filename - - -def test_create_output_product( - tmp_path, - unw_filename, - conncomp_filename, - tcorr_filename, - spatial_corr_filename, -): - output_name = tmp_path / "output_product.nc" - - create_output_product( - unw_filename=unw_filename, - conncomp_filename=conncomp_filename, - tcorr_filename=tcorr_filename, - spatial_corr_filename=spatial_corr_filename, - output_name=output_name, - corrections={}, - )