Merge pull request #4 from Huracan-project/Stella

Add csv loading option

huracanpy/__init__.py

@@ -1,12 +1,12 @@
 """huracanpy - A python package for working with various forms of feature tracking data"""
 
 __version__ = "0.1.0"
-__author__ = "Leo Saffin <[email protected]>, Stella Bourdin, Kelvin Ng "
+__author__ = "Leo Saffin <[email protected]>, Stella Bourdin <[email protected]>, Kelvin Ng "
 __all__ = ["load"]
 
 import pathlib
 
-from ._tracker_specific import TRACK
+from ._tracker_specific import TRACK, csv
 
 
 here = pathlib.Path(__file__).parent
@@ -16,9 +16,16 @@
     testdata_dir / "tr_trs_pos.2day_addT63vor_addmslp_add925wind_add10mwind.tcident.new"
 )
 
+example_csv_file = str(
+    testdata_dir / "sample.csv"
+)
+
 
 def load(filename, tracker=None, **kwargs):
     if tracker.lower() == "track":
         return TRACK.load(filename, **kwargs)
+    if tracker.lower() in ["csv", "te", "tempestextremes", "uz"]:
+        return csv.load(filename)
     else:
         raise ValueError(f"Tracker {tracker} unsupported or misspelled")
+

huracanpy/_tracker_specific/csv.py

@@ -0,0 +1,74 @@
+"""
+Module to load tracks stored as csv files, including TempestExtremes output.
+"""
+
+import numpy as np
+import xarray as xr
+import pandas as pd
+
+def get_time(year, month, day, hour):
+    """
+    Get np.datetime64 array corresponding to year, month, day and hour arrays
+
+    Parameters
+    ----------
+    year (np.array or pd.Series)
+    month (np.array or pd.Series)
+    day (np.array or pd.Series)
+    hour (np.array or pd.Series)
+
+    Returns
+    -------
+    np.array or pd.Series
+        The corresponding np.datetime64
+    """
+    time = pd.to_datetime(
+        year.astype(str)
+        + "-"
+        + month.astype(str)
+        + "-"
+        + day.astype(str)
+        + " "
+        + hour.astype(str)
+        + ":00"
+    )
+    return time
+
+
+def load(filename,):
+    """Load csv tracks data as an xarray.Dataset
+    These tracks may come from TempestExtremes StitchNodes, or any other source.
+
+    Parameters
+    ----------
+    filename : str
+        The file must contain at least longitude, latitude, time and track ID. 
+            - longitude and latitude can be named that, or lon and lat.
+            - time must be defined by four columns : year, month, day, hour 
+            - track ID must be within a column named track_id.
+
+    Returns
+    -------
+    xarray.Dataset
+    """
+
+    ## Read file
+    tracks = pd.read_csv(filename)
+    if tracks.columns.str[0][1] == " ": # Sometimes columns names are read starting with a space, which we remove
+        tracks = tracks.rename(columns={c: c[1:] for c in tracks.columns[1:]})
+    tracks = tracks.rename({"longitude":"lon", "latitude":"lat"}) # Rename lon & lat columns if necessary
+
+    ## Geographical attributes
+    tracks.loc[tracks.lon < 0, "lon"] += 360 # Longitude are converted to [0,360] if necessary
+    # TODO : Move it (^) to the wrapper level ?
+    #tracks["hemisphere"] = np.where(tracks.lat > 0, "N", "S")
+    # TODO : Determine basin (wrapper level?)
+
+    ## Temporal attributes
+    tracks["time"] = get_time(tracks.year, tracks.month, tracks.day, tracks.hour)
+    # TODO : Determine season (wrapper level?)
+
+    # Output xr dataset
+    return tracks.to_xarray().rename({"index":"obs"})
+
+

huracanpy/example_data/sample.csv

@@ -0,0 +1,100 @@
+track_id, year, month, day, hour, i, j, lon, lat, slp, zs, wind10
+0, 1980, 1, 6, 6, 482, 417, 120.500000, -14.250000, 9.987638e+04, -1.071180e+01, 1.464815e+01
+0, 1980, 1, 6, 12, 476, 419, 119.000000, -14.750000, 9.981100e+04, -1.610522e+01, 1.398848e+01
+0, 1980, 1, 6, 18, 476, 420, 119.000000, -15.000000, 9.953694e+04, -4.020874e+01, 1.369575e+01
+0, 1980, 1, 7, 0, 477, 420, 119.250000, -15.000000, 9.941456e+04, -5.043206e+01, 1.797812e+01
+0, 1980, 1, 7, 6, 478, 422, 119.500000, -15.500000, 9.917319e+04, -7.229797e+01, 1.722728e+01
+0, 1980, 1, 7, 12, 475, 424, 118.750000, -16.000000, 9.911612e+04, -7.817557e+01, 1.637649e+01
+0, 1980, 1, 7, 18, 474, 426, 118.500000, -16.500000, 9.888225e+04, -9.791051e+01, 1.840417e+01
+0, 1980, 1, 8, 0, 473, 427, 118.250000, -16.750000, 9.813756e+04, -1.645126e+02, 2.211769e+01
+0, 1980, 1, 8, 6, 473, 430, 118.250000, -17.500000, 9.780431e+04, -1.938985e+02, 2.554702e+01
+0, 1980, 1, 8, 12, 473, 430, 118.250000, -17.500000, 9.821738e+04, -1.573104e+02, 2.342027e+01
+0, 1980, 1, 8, 18, 475, 431, 118.750000, -17.750000, 9.810538e+04, -1.670528e+02, 2.355895e+01
+0, 1980, 1, 9, 0, 477, 433, 119.250000, -18.250000, 9.757319e+04, -2.135222e+02, 2.536765e+01
+0, 1980, 1, 9, 6, 477, 436, 119.250000, -19.000000, 9.713475e+04, -2.492251e+02, 2.473392e+01
+0, 1980, 1, 9, 12, 479, 437, 119.750000, -19.250000, 9.713338e+04, -2.525460e+02, 2.639518e+01
+0, 1980, 1, 9, 18, 480, 438, 120.000000, -19.500000, 9.696419e+04, -2.690853e+02, 2.594052e+01
+0, 1980, 1, 10, 0, 480, 440, 120.000000, -20.000000, 9.722312e+04, -2.483882e+02, 2.525727e+01
+0, 1980, 1, 10, 6, 478, 441, 119.500000, -20.250000, 9.732125e+04, -2.369373e+02, 2.478705e+01
+0, 1980, 1, 10, 12, 477, 443, 119.250000, -20.750000, 9.742400e+04, -2.268684e+02, 2.252732e+01
+0, 1980, 1, 10, 18, 473, 445, 118.250000, -21.250000, 9.742656e+04, -2.230690e+02, 2.084525e+01
+0, 1980, 1, 11, 0, 470, 447, 117.500000, -21.750000, 9.855894e+04, -1.264948e+02, 1.687273e+01
+0, 1980, 1, 11, 6, 468, 452, 117.000000, -23.000000, 9.871950e+04, -1.109990e+02, 1.673304e+01
+0, 1980, 1, 11, 12, 468, 455, 117.000000, -23.750000, 9.889306e+04, -9.445285e+01, 1.693710e+01
+0, 1980, 1, 11, 18, 467, 461, 116.750000, -25.250000, 9.888469e+04, -9.741785e+01, 1.757735e+01
+0, 1980, 1, 12, 0, 467, 467, 116.750000, -26.750000, 9.940956e+04, -5.202582e+01, 1.595991e+01
+0, 1980, 1, 12, 6, 470, 472, 117.500000, -28.000000, 9.960675e+04, -3.346035e+01, 1.536911e+01
+0, 1980, 1, 12, 12, 477, 478, 119.250000, -29.500000, 9.987350e+04, -1.217747e+01, 1.355582e+01
+0, 1980, 1, 12, 18, 484, 483, 121.000000, -30.750000, 9.981912e+04, -1.511865e+01, 1.351684e+01
+0, 1980, 1, 13, 0, 494, 486, 123.500000, -31.500000, 1.003131e+05, 2.642408e+01, 1.250123e+01
+0, 1980, 1, 13, 6, 510, 487, 127.500000, -31.750000, 1.001848e+05, 1.529015e+01, 1.311584e+01
+0, 1980, 1, 13, 12, 521, 486, 130.250000, -31.500000, 1.003802e+05, 3.307928e+01, 1.147507e+01
+0, 1980, 1, 13, 18, 525, 487, 131.250000, -31.750000, 1.004768e+05, 4.133395e+01, 1.023079e+01
+1, 1980, 1, 7, 0, 598, 446, 149.500000, -21.500000, 9.962706e+04, -3.296264e+01, 1.268272e+01
+1, 1980, 1, 7, 6, 606, 453, 151.500000, -23.250000, 9.912119e+04, -7.643916e+01, 1.866169e+01
+1, 1980, 1, 7, 12, 616, 458, 154.000000, -24.500000, 9.929812e+04, -6.096099e+01, 2.071399e+01
+1, 1980, 1, 7, 18, 625, 464, 156.250000, -26.000000, 9.901400e+04, -8.464599e+01, 1.896645e+01
+1, 1980, 1, 8, 0, 634, 471, 158.500000, -27.750000, 9.911431e+04, -7.791729e+01, 2.028473e+01
+1, 1980, 1, 8, 6, 638, 477, 159.500000, -29.250000, 9.886381e+04, -9.741531e+01, 1.948820e+01
+1, 1980, 1, 8, 12, 640, 483, 160.000000, -30.750000, 9.845238e+04, -1.335337e+02, 2.121326e+01
+1, 1980, 1, 8, 18, 640, 487, 160.000000, -31.750000, 9.801238e+04, -1.726721e+02, 2.378614e+01
+1, 1980, 1, 9, 6, 640, 499, 160.000000, -34.750000, 9.809550e+04, -1.648724e+02, 2.364721e+01
+1, 1980, 1, 9, 18, 633, 511, 158.250000, -37.750000, 9.832544e+04, -1.447352e+02, 2.203883e+01
+1, 1980, 1, 10, 0, 624, 516, 156.000000, -39.000000, 9.839538e+04, -1.397746e+02, 2.155125e+01
+1, 1980, 1, 10, 6, 617, 519, 154.250000, -39.750000, 9.815875e+04, -1.587136e+02, 1.929576e+01
+1, 1980, 1, 10, 12, 613, 523, 153.250000, -40.750000, 9.842700e+04, -1.365779e+02, 1.947424e+01
+1, 1980, 1, 10, 18, 611, 528, 152.750000, -42.000000, 9.837581e+04, -1.390986e+02, 1.862356e+01
+1, 1980, 1, 11, 0, 610, 533, 152.500000, -43.250000, 9.846994e+04, -1.315279e+02, 1.771547e+01
+1, 1980, 1, 11, 6, 610, 538, 152.500000, -44.500000, 9.848375e+04, -1.292840e+02, 1.566105e+01
+1, 1980, 1, 11, 12, 612, 542, 153.000000, -45.500000, 9.876006e+04, -1.053346e+02, 1.448115e+01
+1, 1980, 1, 11, 18, 615, 545, 153.750000, -46.250000, 9.885569e+04, -9.642397e+01, 1.314373e+01
+1, 1980, 1, 12, 0, 619, 546, 154.750000, -46.500000, 9.933831e+04, -5.582296e+01, 1.139226e+01
+1, 1980, 1, 12, 6, 624, 547, 156.000000, -46.750000, 9.962050e+04, -3.181662e+01, 1.045837e+01
+2, 1980, 1, 17, 6, 221, 435, 55.250000, -18.750000, 1.005510e+05, 4.869846e+01, 1.151261e+01
+2, 1980, 1, 17, 18, 217, 433, 54.250000, -18.250000, 1.005613e+05, 4.958588e+01, 1.295175e+01
+2, 1980, 1, 18, 6, 211, 433, 52.750000, -18.250000, 1.006245e+05, 5.498806e+01, 1.491472e+01
+2, 1980, 1, 19, 0, 216, 422, 54.000000, -15.500000, 1.004833e+05, 4.257021e+01, 1.536913e+01
+2, 1980, 1, 19, 6, 220, 425, 55.000000, -16.250000, 1.006609e+05, 5.827243e+01, 1.263563e+01
+2, 1980, 1, 20, 6, 208, 430, 52.000000, -17.500000, 1.004542e+05, 4.016698e+01, 1.048888e+01
+2, 1980, 1, 20, 12, 204, 431, 51.000000, -17.750000, 1.003288e+05, 2.920397e+01, 1.216490e+01
+2, 1980, 1, 20, 18, 208, 428, 52.000000, -17.000000, 1.003032e+05, 2.696300e+01, 1.300553e+01
+2, 1980, 1, 21, 0, 206, 431, 51.500000, -17.750000, 1.001099e+05, 9.620783e+00, 1.301741e+01
+2, 1980, 1, 21, 6, 203, 434, 50.750000, -18.500000, 1.002412e+05, 2.086479e+01, 1.428573e+01
+2, 1980, 1, 21, 12, 201, 434, 50.250000, -18.500000, 1.001051e+05, 9.146439e+00, 1.183709e+01
+2, 1980, 1, 21, 18, 202, 434, 50.500000, -18.500000, 1.002229e+05, 1.949262e+01, 1.459647e+01
+2, 1980, 1, 22, 0, 203, 436, 50.750000, -19.000000, 1.001661e+05, 1.460683e+01, 1.186565e+01
+2, 1980, 1, 22, 6, 199, 439, 49.750000, -19.750000, 1.002580e+05, 2.275181e+01, 1.112769e+01
+2, 1980, 1, 22, 12, 200, 438, 50.000000, -19.500000, 9.995381e+04, -3.940494e+00, 1.142356e+01
+2, 1980, 1, 22, 18, 202, 439, 50.500000, -19.750000, 1.001214e+05, 1.050467e+01, 1.450923e+01
+2, 1980, 1, 23, 0, 204, 440, 51.000000, -20.000000, 9.995281e+04, -4.096187e+00, 1.259182e+01
+2, 1980, 1, 23, 6, 207, 442, 51.750000, -20.500000, 1.000392e+05, 3.474611e+00, 1.299474e+01
+2, 1980, 1, 23, 12, 209, 445, 52.250000, -21.250000, 9.985381e+04, -1.271542e+01, 1.143798e+01
+2, 1980, 1, 23, 18, 212, 446, 53.000000, -21.500000, 9.996569e+04, -2.803262e+00, 1.192467e+01
+2, 1980, 1, 24, 0, 213, 448, 53.250000, -22.000000, 9.956275e+04, -3.810565e+01, 1.733382e+01
+2, 1980, 1, 24, 6, 211, 445, 52.750000, -21.250000, 9.961062e+04, -3.360746e+01, 1.669525e+01
+2, 1980, 1, 24, 12, 218, 446, 54.500000, -21.500000, 9.939019e+04, -5.226105e+01, 1.883360e+01
+2, 1980, 1, 24, 18, 212, 444, 53.000000, -21.000000, 9.940650e+04, -5.150117e+01, 1.757754e+01
+2, 1980, 1, 25, 0, 215, 442, 53.750000, -20.500000, 9.905669e+04, -8.210120e+01, 1.863198e+01
+2, 1980, 1, 25, 6, 214, 441, 53.500000, -20.250000, 9.904144e+04, -8.357072e+01, 1.817620e+01
+2, 1980, 1, 25, 12, 213, 441, 53.250000, -20.250000, 9.876275e+04, -1.082670e+02, 1.887632e+01
+2, 1980, 1, 25, 18, 213, 442, 53.250000, -20.500000, 9.870594e+04, -1.128646e+02, 2.107328e+01
+2, 1980, 1, 26, 0, 211, 443, 52.750000, -20.750000, 9.842488e+04, -1.374308e+02, 2.089620e+01
+2, 1980, 1, 26, 6, 210, 445, 52.500000, -21.250000, 9.842938e+04, -1.368288e+02, 2.105174e+01
+2, 1980, 1, 26, 12, 209, 446, 52.250000, -21.500000, 9.821531e+04, -1.560316e+02, 2.036689e+01
+2, 1980, 1, 26, 18, 210, 446, 52.500000, -21.500000, 9.829106e+04, -1.485572e+02, 2.104506e+01
+2, 1980, 1, 27, 0, 213, 446, 53.250000, -21.500000, 9.805712e+04, -1.694464e+02, 2.137289e+01
+2, 1980, 1, 27, 6, 217, 447, 54.250000, -21.750000, 9.818400e+04, -1.582613e+02, 2.175065e+01
+2, 1980, 1, 27, 12, 222, 449, 55.500000, -22.250000, 9.783506e+04, -1.895754e+02, 2.323915e+01
+2, 1980, 1, 27, 18, 227, 454, 56.750000, -23.500000, 9.770181e+04, -2.016107e+02, 2.226480e+01
+2, 1980, 1, 28, 0, 233, 456, 58.250000, -24.000000, 9.743356e+04, -2.245587e+02, 2.397522e+01
+2, 1980, 1, 28, 6, 238, 461, 59.500000, -25.250000, 9.732900e+04, -2.342961e+02, 2.592217e+01
+2, 1980, 1, 28, 12, 237, 468, 59.250000, -27.000000, 9.750131e+04, -2.196940e+02, 2.621428e+01
+2, 1980, 1, 28, 18, 235, 475, 58.750000, -28.750000, 9.754275e+04, -2.140831e+02, 2.564483e+01
+2, 1980, 1, 29, 0, 233, 479, 58.250000, -29.750000, 9.724831e+04, -2.414689e+02, 2.315311e+01
+2, 1980, 1, 29, 6, 231, 486, 57.750000, -31.500000, 9.751525e+04, -2.178650e+02, 2.376217e+01
+2, 1980, 1, 29, 12, 230, 494, 57.500000, -33.500000, 9.747712e+04, -2.202573e+02, 2.311012e+01
+2, 1980, 1, 29, 18, 229, 501, 57.250000, -35.250000, 9.761275e+04, -2.072094e+02, 2.255570e+01
+2, 1980, 1, 30, 0, 230, 509, 57.500000, -37.250000, 9.732038e+04, -2.325650e+02, 2.319099e+01
+2, 1980, 1, 30, 6, 234, 517, 58.500000, -39.250000, 9.740294e+04, -2.254557e+02, 2.369188e+01
+2, 1980, 1, 30, 12, 241, 528, 60.250000, -42.000000, 9.747119e+04, -2.185280e+02, 2.395939e+01
+2, 1980, 1, 30, 18, 249, 542, 62.250000, -45.500000, 9.753669e+04, -2.114699e+02, 2.340470e+01

pyproject.toml

@@ -3,7 +3,7 @@ name = "huracanpy"
 version = "0.1.0"
 authors = [
   { name="Leo Saffin", email="[email protected]" },
-  { name="Stella Bourdin" },
+  { name="Stella Bourdin", email="[email protected]" },
   { name="Kelvin Ng" },
 ]
 description = "A python package for working with various forms of feature tracking data"
@@ -20,3 +20,17 @@ build-backend = "setuptools.build_meta"
 
 [tool.setuptools.package-data]
 huracanpy = ["example_data/*"]
+
+
+# - How to distribute the package
+
+# 0/ Applying black formatting
+
+# 1/ Generate the distribution
+# 1a. Make sure that build is up-to-date: `pip install --upgrade build`(add the --user option if necessary)
+# 1b. Run python -m build in the package's root directory
+# -> Two new files are created in the dist folder, a wheel (.whl) and a tarball (.tar.gz), with the the version code in the present file
+
+# 2/ Distribute the distribution of PyPI
+# 2a. Make sure that twine is up-to-date `pip install (--user) --upgrade twine`
+# 2b. Upload the package: `python -m twine upload --repository pypi dist/*x.x.x*` (x.x.x = version number)

tests/test_huracanpy.py

@@ -4,3 +4,9 @@
 def test_load_track():
     data = huracanpy.load(huracanpy.example_TRACK_file, tracker="TRACK")
     assert len(data.groupby("track_id")) == 2
+
+def test_load_csv():
+    data = huracanpy.load(huracanpy.example_csv_file, tracker = "tempestextremes")
+    assert len(data) == 13
+    assert len(data.obs) == 99
+    assert len(data.groupby("track_id")) == 3