Merge branch 'add_artifical-information_filter' of https://github.com…

…/Ishaanj18/xbitinfo into pr/234
observingClouds · Apr 16, 2024 · c462b6e · c462b6e
2 parents 4adee89 + 243c017
commit c462b6e
Show file tree

Hide file tree

Showing 19 changed files with 683 additions and 103 deletions.
diff --git a/.flake8 b/.flake8
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Miniconda
-        uses: conda-incubator/[email protected].1
+        uses: conda-incubator/[email protected].3
         with:
           # installer-url: https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh
           installer-url: https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh
@@ -74,7 +74,7 @@ jobs:
           cp benchmarks.log .asv/results/
         working-directory: ${{ env.ASV_DIR }}
 
-      - uses: actions/upload-artifact@v3.1.3
+      - uses: actions/upload-artifact@v4.3.1
         if: always()
         with:
           name: asv-benchmark-results-${{ runner.os }}

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -33,7 +33,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up conda
-        uses: conda-incubator/[email protected].1
+        uses: conda-incubator/[email protected].3
         with:
           auto-update-conda: false
           channels: conda-forge
@@ -60,7 +60,7 @@ jobs:
         shell: 'bash -l {0}'
     steps:
       - uses: actions/[email protected]
-      - uses: conda-incubator/[email protected].1
+      - uses: conda-incubator/[email protected].3
         with:
           channels: conda-forge
           miniforge-variant: Mambaforge
@@ -91,7 +91,7 @@ jobs:
     steps:
       - uses: actions/[email protected]
       - name: Set up conda
-        uses: conda-incubator/[email protected].1
+        uses: conda-incubator/[email protected].3
         with:
           auto-update-conda: false
           channels: conda-forge
@@ -134,11 +134,11 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.1.0
         with:
           python-version: '3.11'
       - name: Set up Julia
-        uses: julia-actions/setup-julia@v1.9.4
+        uses: julia-actions/setup-julia@v2.0.0
         with:
           version: 1.7.1
       - name: Install dependencies

diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml
@@ -16,7 +16,7 @@ jobs:
     - uses: actions/[email protected]
 
     - name: Set up Python
-      uses: actions/setup-python@v5.0.0
+      uses: actions/setup-python@v5.1.0
       with:
         python-version: "3.10"
 
@@ -45,7 +45,7 @@ jobs:
 
     - name: Publish a Python distribution to PyPI
       if: success() && github.event_name == 'release'
-      uses: pypa/[email protected].11
+      uses: pypa/[email protected].14
       with:
         user: __token__
         password: ${{ secrets.PYPI_PASSWORD }}
diff --git a/.github/workflows/updater.yaml b/.github/workflows/updater.yaml
@@ -21,3 +21,4 @@ jobs:
         with:
           # [Required] Access token with `workflow` scope.
           token: ${{ secrets.WORKFLOW_SECRET }}
+          pull_request_branch: gh-actions-update
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,62 +2,30 @@ ci:
     autoupdate_schedule: quarterly
 
 
-# https://pre-commit.com/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
       - id: debug-statements
       - id: mixed-line-ending
-    # This wants to go before isort & flake8
-  - repo: https://github.com/PyCQA/autoflake
-    rev: "v2.0.2"
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.2
     hooks:
-      - id: autoflake # isort should run before black as black sometimes tweaks the isort output
-        args: ["--in-place", "--ignore-init-module-imports"]
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        args: ["--profile", "black"]
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
-    hooks:
-      - id: pyupgrade
-        args:
-          - "--py38-plus"
-  # https://github.com/python/black#version-control-integration
+      # run the linter
+      - id: ruff
+        args: [--fix, --ignore-noqa]
+
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 24.3.0
     hooks:
       - id: black
       - id: black-jupyter
+
   - repo: https://github.com/keewis/blackdoc
-    rev: v0.3.8
+    rev: v0.3.9
     hooks:
       - id: blackdoc
         exclude: docs/index.rst
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.1.1
-    hooks:
-      - id: mypy
-        exclude: "properties|asv_bench"
-        # This is slow and so we take it out of the fast-path; requires passing
-        # `--hook-stage manual` to pre-commit
-        stages: [manual]
-        additional_dependencies: [
-            # Type stubs
-            types-python-dateutil,
-            types-pkg_resources,
-            types-PyYAML,
-            types-pytz,
-            typing-extensions==3.10.0.0,
-            numpy,
-          ]
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -5,6 +5,7 @@ CHANGELOG
 X.X.X (unreleased)
 ------------------
 
+* Add support for additional datatypes in :py:func:`xbitinfo.xbitinfo.plot_bitinformation` (:pr:`218`, :issue:`168`) `Hauke Schulz`_.
 * Drop python 3.8 support and add python 3.11 (:pr:`175`) `Hauke Schulz`_.
 * Implement basic retrieval of bitinformation in python as alternative to julia implementation (:pr:`156`, :issue:`155`, :pr:`126`, :issue:`125`) `Hauke Schulz`_ with helpful comments from `Milan Klöwer`_.
 * Make julia binding to BitInformation.jl optional (:pr:`153`, :issue:`151`) `Aaron Spring`_.

diff --git a/docs/ArtificialInformation_Filter.ipynb b/docs/ArtificialInformation_Filter.ipynb
@@ -1,5 +1,13 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "38ac6d1a",
+   "metadata": {},
+   "source": [
+    "<span style=\"color:red\">**<<<<<<< local**</span>"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "895e1d5a",
@@ -64,6 +72,86 @@
     "### Functions to encode and decode"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "69543b4c",
+   "metadata": {},
+   "source": [
+    "<span style=\"color:red\">**=======**</span>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1842f792",
+   "metadata": {},
+   "source": [
+    "# Artificial information filtering\n",
+    "\n",
+    "In simple terms the bitinformation is retrieved by checking how variable a bit pattern is. However, this approach cannot distinguish between actual information content and artifical information content. By studying the distribution of the information content the user can often identify clear cut-offs of real information content and artificial information content.\n",
+    "\n",
+    "The following example shows how such a separation of real information and artificial information can look like. To do so, artificial information is artificially added to an example dataset by applying linear quantization. Linear quantization is often applied to climate datasets (e.g. ERA5) and needs to be accounted for in order to retrieve meaningful bitinformation content. An algorithm that aims at detecting this artificial information itself is introduced."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bb998fbb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import xarray as xr\n",
+    "import xbitinfo as xb\n",
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "32ac97e0",
+   "metadata": {},
+   "source": [
+    "## Loading example dataset\n",
+    "We use here the openly accessible CONUS dataset. The dataset is available at full precision."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9639a618",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = xr.open_zarr(\n",
+    "    \"s3://hytest/conus404/conus404_monthly.zarr\",\n",
+    "    storage_options={\n",
+    "        \"anon\": True,\n",
+    "        \"requester_pays\": False,\n",
+    "        \"client_kwargs\": {\"endpoint_url\": \"https://usgs.osn.mghpcc.org\"},\n",
+    "    },\n",
+    ")\n",
+    "# selecting water vapor mixing ratio at 2 meters\n",
+    "data = ds[\"ACSWDNT\"]\n",
+    "# select subset of data for demonstration purposes\n",
+    "chunk = data.isel(time=slice(0, 2), y=slice(0, 1015), x=slice(0, 1050))\n",
+    "chunk"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3d735e4b",
+   "metadata": {},
+   "source": [
+    "## Creating dataset copy with artificial information\n",
+    "### Functions to encode and decode"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0d30feaa",
+   "metadata": {},
+   "source": [
+    "<span style=\"color:red\">**>>>>>>> remote**</span>"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,