Always override default NaNs when loading from CSV and include all defaults from pandas apart from NA

Author: leosaffin

huracanpy/_data/_csv.py

@@ -7,11 +7,35 @@
 
 from .. import utils
 
+# All values recognised as NaN by pandas.read_csv, except "NA" which we want to load
+# normally because it is a basin, and added "" to interpret empty entries as NaN
+pandas_na_values = [
+    " ",
+    "#N/A",
+    "#N/A N/A",
+    "#NA",
+    "-1.#IND",
+    "-1.#QNAN",
+    "-NaN",
+    "-nan",
+    "1.#IND",
+    "1.#QNAN",
+    "<NA>",
+    "N/A",
+    "NULL",
+    "NaN",
+    "None",
+    "n/a",
+    "nan",
+    "null ",
+    "",
+]
+
 
 def load(
     filename,
     load_function=pd.read_csv,
-    read_csv_kws=dict(),
+    **kwargs,
 ):
     """Load csv tracks data as an xarray.Dataset
     These tracks may come from TempestExtremes StitchNodes, or any other source.
@@ -24,13 +48,23 @@ def load(
             - time must be defined a single `time`column or by four columns : year, month, day, hour
             - track ID must be within a column named track_id.
 
+    load_function : callable
+        One of the load functions in pandas
+
+    **kwargs
+        Remaining keywords are passed to the pandas
+
     Returns
     -------
     xarray.Dataset
     """
+    # Update keywords with extra defaults for dealing with "NA" as basin not nan
+    # Put kwargs second in this statement, so it can override defaults
+    if load_function is pd.read_csv:
+        kwargs = {**dict(na_values=pandas_na_values, keep_default_na=False), **kwargs}
 
     ## Read file
-    tracks = load_function(filename, **read_csv_kws)
+    tracks = load_function(filename, **kwargs)
     if (
         tracks.columns.str[0][1] == " "
     ):  # Sometimes columns names are read starting with a space, which we remove

huracanpy/_data/_load.py

@@ -116,6 +116,10 @@ def load(
         * CSV file - :func:`pandas.read_csv`
         * parquet file - :func:`pandas.read_parquet`
 
+        For CSV files pandas interprets "NA" as `nan` by default, which is overriden in
+        this function. To restore the pandas default behavious set
+        :code:`keep_default_NA=True` and :code:`na_values=[]`
+
     Returns
     -------
     xarray.Dataset
@@ -145,7 +149,7 @@ def load(
                 calendar=track_calendar,
             )
         elif source.lower() in ["csv", "uz"]:
-            data = _csv.load(filename)
+            data = _csv.load(filename, **kwargs)
         elif source.lower() in ["te", "tempest", "tempestextremes"]:
             data = _tempestextremes.load(
                 filename,
@@ -163,13 +167,13 @@ def load(
                     filename = "ibtracs.csv"
 
                 with ibtracs.online(ibtracs_subset, filename, ibtracs_clean) as f:
-                    data = _csv.load(
-                        f,
-                        read_csv_kws=dict(
+                    # Put IBTrACS specific arguments to read_csv second, so it
+                    # overwrites any arguments passed
+                    kwargs = {
+                        **kwargs,
+                        **dict(
                             header=0,
                             skiprows=[1],
-                            na_values=["", " "],
-                            keep_default_na=False,
                             converters={
                                 "SID": str,
                                 "SEASON": int,
@@ -179,14 +183,20 @@ def load(
                                 "LAT": float,
                             },
                         ),
+                    }
+                    return load(
+                        filename=f,
+                        source="csv",
+                        rename=rename,
+                        add_info=add_info,
+                        **kwargs,
                     )
             else:
-                data = _csv.load(
-                    ibtracs.offline(ibtracs_subset),
-                    read_csv_kws=dict(
-                        na_values=["", " "],
-                        keep_default_na=False,
-                    ),
+                return load(
+                    filename=ibtracs.offline(ibtracs_subset),
+                    rename=rename,
+                    add_info=add_info,
+                    **kwargs,
                 )
         else:
             raise ValueError(f"Source {source} unsupported or misspelled")