Add ability to add features to existing track without retracking

.github/workflows/check_notebooks.yml

@@ -10,21 +10,12 @@ jobs:
       - name: check out repository code
         uses: actions/checkout@v3
       - name: set up conda environment
-        uses: conda-incubator/setup-miniconda@v2
+        uses: mamba-org/setup-micromamba@v1
         with:
-          miniforge-version: latest
-          miniforge-variant: mambaforge
-          channel-priority: strict
-          channels: conda-forge
-          show-channel-urls: true
-          use-only-tar-bz2: true
-          auto-update-conda: true
-          auto-activate-base: false
-          activate-environment: notebook-env
-      - name: Install tobac dependencies
-        run: |
-          mamba install -c conda-forge --yes ffmpeg gcc jupyter pytables
-          mamba install -c conda-forge --yes --file example_requirements.txt
+          environment-file: environment-examples.yml
+          generate-run-shell: true
+          cache-environment: true
+          cache-downloads: true
       - name: Install tobac
         run: |
           pip install .

.github/workflows/codecov-CI.yml

@@ -7,24 +7,20 @@ jobs:
     runs-on: ubuntu-latest
     env:
       OS: ubuntu-latest
-      PYTHON: "3.9"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       # Similar to MetPy install-conda action
       - name: Set up conda
-        uses: conda-incubator/setup-miniconda@v2
+        uses: mamba-org/setup-micromamba@v1
         with:
-          miniforge-version: latest
-          miniforge-variant: mambaforge
-          channel-priority: strict
-          channels: conda-forge
-          show-channel-urls: true
-          use-only-tar-bz2: true
+          environment-file: environment-ci.yml
+          generate-run-shell: true
+          cache-environment: true
+          cache-downloads: true
 
-      - name: Install dependencies and generate report
-        shell: bash -l {0}
+      - name: Generate report
+        shell: micromamba-shell {0}
         run:
-          mamba install --quiet --yes --file requirements.txt coverage pytest-cov &&
           python -m coverage run -m pytest --cov=./ --cov-report=xml
       - name: Upload Coverage to Codecov
         uses: codecov/codecov-action@v4

.github/workflows/pylint.yml

@@ -14,21 +14,18 @@ jobs:
         shell: bash -l {0}
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up conda
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Set up mamba
+        uses: mamba-org/setup-micromamba@v1
         with:
-          miniforge-version: latest
-          miniforge-variant: mambaforge
-          channel-priority: strict
-          channels: conda-forge
-          show-channel-urls: true
-          use-only-tar-bz2: true
+          environment-file: environment-ci.yml
+          generate-run-shell: true
+          cache-environment: true
+          cache-downloads: true
 
-      - name: Install tobac and pylint
+      - name: Install tobac
         run: |
-          mamba install --yes pylint
           pip install .
 
       - name: Store the PR branch
@@ -37,7 +34,7 @@ jobs:
         id: git
 
       - name: Checkout RC branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.base_ref }}
 
@@ -47,7 +44,7 @@ jobs:
         id: main_score
 
       - name: Checkout PR branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: "${{ steps.git.outputs.SHA }}"  
 

environment-ci.yml

@@ -11,7 +11,10 @@ dependencies:
   - iris
   - xarray
   - cartopy
-  - trackpy
+  - trackpy>=0.6.1
   - pytest
   - typing_extensions
   - black
+  - coverage
+  - pytest-cov
+  - pylint

environment-examples.yml

@@ -0,0 +1,29 @@
+name: tobac-dev
+channels:
+  - conda-forge
+dependencies:
+  - numpy
+  - scipy
+  - scikit-image
+  - scikit-learn
+  - pandas
+  - matplotlib
+  - iris
+  - xarray
+  - cartopy
+  - trackpy>=0.6.1
+  - pytest
+  - typing_extensions
+  - black
+  - jupyter
+  - notebook
+  - pytables
+  - s3fs
+  - arm_pyart
+  - seaborn
+  - h5netcdf
+  - typing_extensions
+  - rioxarray
+  - numba
+  - dask
+  -

tobac/testing.py

@@ -8,6 +8,7 @@
 import pandas as pd
 from collections import Counter
 from .utils import periodic_boundaries as pbc_utils
+from typing import Union, Optional, Literal, Iterable
 
 
 def make_simple_sample_data_2D(data_type="iris"):
@@ -815,37 +816,42 @@
 
 
 def generate_single_feature(
-    start_h1,
-    start_h2,
-    start_v=None,
-    spd_h1=1,
-    spd_h2=1,
-    spd_v=1,
-    min_h1=0,
-    max_h1=None,
-    min_h2=0,
-    max_h2=None,
-    num_frames=1,
-    dt=datetime.timedelta(minutes=5),
-    start_date=datetime.datetime(2022, 1, 1, 0),
-    PBC_flag="none",
-    frame_start=0,
-    feature_num=1,
-    feature_size=None,
-    threshold_val=None,
+    start_h1: Union[float, Iterable[float]],
+    start_h2: Union[float, Iterable[float]],
+    start_v: Optional[Union[float, Iterable[float]]] = None,
+    spd_h1: float = 1,
+    spd_h2: float = 1,
+    spd_v: float = 1,
+    min_h1: float = 0,
+    max_h1: Optional[float] = None,
+    min_h2: float = 0,
+    max_h2: Optional[float] = None,
+    num_frames: int = 1,
+    dt: datetime.timedelta = datetime.timedelta(minutes=5),
+    start_date: datetime.datetime = datetime.datetime(2022, 1, 1, 0),
+    PBC_flag: Literal["none", "hdim_1", "hdim_2", "both"] = "none",
+    frame_start: int = 0,
+    feature_num: int = 1,
+    feature_size: Optional[int] = None,
+    threshold_val: Optional[float] = None,
 ):
     """Function to generate a dummy feature dataframe to test the tracking functionality
 
     Parameters
     ----------
-    start_h1: float
-        Starting point of the feature in hdim_1 space
+    start_h1: float or iterable of floats
+        Starting point of the feature in hdim_1 space if float
+        or all h1 points if iterable. If iterable is passed,
+        all other parameters are ignored.
 
-    start_h2: float
-        Starting point of the feature in hdim_2 space
+    start_h2: float or iterable of floats
+        Starting point of the feature in hdim_2 space if float
+        or all h2 points if iterable
 
     start_v: float, optional
-        Starting point of the feature in vdim space (if 3D). For 2D, set to None.
+        Starting point of the feature in vdim space (if 3D) if float
+        or all vertical points if iterable.
+        For 2D, set to None.
         Default is None
 
     spd_h1: float, optional
@@ -914,38 +920,101 @@
     if max_h1 is None or max_h2 is None:
         raise ValueError("Max coords must be specified.")
 
-    out_list_of_dicts = list()
-    curr_h1 = start_h1
-    curr_h2 = start_h2
-    curr_v = start_v
+    h1_list = []
+    h2_list = []
+    v_list = []
+
+    # check if start_h1, start_h2, and start_v are all arrays
+    if not np.isscalar(start_h1) and not np.isscalar(start_h2):
+        if start_v is not None and np.isscalar(start_h2):
+            raise ValueError(
+                "Must specify all start_h1, start_h2, and start_v as arrays."
+            )
+        h1_list = start_h1
+        h2_list = start_h2
+        if start_v is not None:
+            v_list = start_v
+    # we need to make the list of points.
+    else:
+        curr_h1 = start_h1
+        curr_h2 = start_h2
+        curr_v = start_v
+
+        for i in range(num_frames):
+            curr_h1, curr_h2 = get_single_pbc_coordinate(
+                min_h1, max_h1, min_h2, max_h2, curr_h1, curr_h2, PBC_flag
+            )
+
+            h1_list.append(curr_h1)
+            h2_list.append(curr_h2)
+            curr_h1 += spd_h1
+            curr_h2 += spd_h2
+
+            if start_v is not None:
+                v_list.append(curr_v)
+                curr_v += spd_v
+
+    out_list_of_dicts = []
     curr_dt = start_date
-    is_3D = not (start_v is None)
-    for i in range(num_frames):
+    for i in range(len(h1_list)):
+        if h1_list[i] < 0 or h2_list[i] < 0:
+            curr_dt += dt
+            continue
+        if len(v_list) > 0:
+            if v_list[i] < 0:
+                curr_dt += dt
+                continue
         curr_dict = dict()
-        curr_h1, curr_h2 = get_single_pbc_coordinate(
-            min_h1, max_h1, min_h2, max_h2, curr_h1, curr_h2, PBC_flag
-        )
-        curr_dict["hdim_1"] = curr_h1
-        curr_dict["hdim_2"] = curr_h2
+        curr_dict["hdim_1"] = h1_list[i]
+        curr_dict["hdim_2"] = h2_list[i]
         curr_dict["frame"] = frame_start + i
         curr_dict["idx"] = 0
-        if curr_v is not None:
-            curr_dict["vdim"] = curr_v
+        if len(v_list) > 0:
+            curr_dict["vdim"] = v_list[i]
             curr_v += spd_v
         curr_dict["time"] = curr_dt
         curr_dict["feature"] = feature_num + i
         if feature_size is not None:
             curr_dict["num"] = feature_size
         if threshold_val is not None:
             curr_dict["threshold_value"] = threshold_val
-        curr_h1 += spd_h1
-        curr_h2 += spd_h2
         curr_dt += dt
         out_list_of_dicts.append(curr_dict)
 
     return pd.DataFrame.from_dict(out_list_of_dicts)
 
 
+def combine_single_features(
+    feat_list: list[pd.DataFrame], begin_feat_num: int = 1
+) -> pd.DataFrame:
+    """Combines multiple generate_single_feature outputs into one
+    coherent output
+
+    Parameters
+    ----------
+    feat_list
+
+    Returns
+    -------
+
+    """
+
+    time_col_name: str = "time"
+
+    comb_output = pd.concat(feat_list)
+
+    all_times = comb_output[time_col_name].unique()
+    all_times = sorted(all_times)
+    frame_nums = {str(np.datetime64(x, "ns")): i for i, x in enumerate(all_times)}
+    vec_frame_func = np.vectorize(lambda x: frame_nums[str(np.datetime64(x, "ns"))])
+    comb_output["frame"] = vec_frame_func(comb_output[time_col_name])
+    comb_output["idx"] = comb_output.groupby("frame").transform("cumcount")
+    comb_output = comb_output.sort_values(["frame", "idx"], axis=0)
+    feat_col = np.arange(begin_feat_num, begin_feat_num + len(comb_output))
+    comb_output["feature"] = feat_col
+    return comb_output
+
+
 def get_start_end_of_feat(center_point, size, axis_min, axis_max, is_pbc=False):
     """Gets the start and ending points for a feature given a size and PBC
     conditions