Merge master branch changes into refactor_projects

.github/workflows/pythonpackage.yml

@@ -142,8 +142,8 @@ jobs:
         echo "Disk space after cleanup:"
         df -h .
         echo "========================================================="
-        echo Starting VIRES client tests at `date`
-        coverage run -a -m pyspedas.vires.tests.tests
+        #echo Starting VIRES client tests at `date`
+        #coverage run -a -m pyspedas.vires.tests.tests
         echo Starting secs tests at `date`
         coverage run -a -m pyspedas.secs.tests.tests
         echo Starting MTH5 import test at `date`
@@ -239,6 +239,8 @@ jobs:
         coverage run -a -m pyspedas.utilities.tests.download_tests
         echo Starting utilities misc tests at `date`
         coverage run -a -m pyspedas.utilities.tests.misc_tests
+        echo Starting tplot wildcard tests at `date`
+        coverage run -a -m pyspedas.utilities.tests.tplot_wildcard_tests
         echo Starting plotting tests at `date`
         coverage run -a -m pyspedas.utilities.tests.plot_tests
         echo Starting utilities time_tests tests at `date`

pyspedas/analysis/tests/tests.py

@@ -309,23 +309,44 @@ def test_tinterpol(self):
 
     def test_scipy_interp1d(self):
         import scipy
+        import numpy as np
         time_strings_input = np.array(['2018-07-01T13:02:16.892474880',
                                        '2018-07-01T13:02:16.922475008',
                                        '2018-07-01T13:02:16.952474880'])
         values_input = np.array([0.028584518, 0., 0.013626526],dtype=np.float32)
-        time_strings_interp_to = np.array(['2018-07-01T13:02:16.922475008'])
 
         input_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_input])
-        interp_to_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_interp_to])
+        interp_to_times_npdt64 = np.array(input_times_npdt64[1])
 
         input_times_float64 = input_times_npdt64.astype(np.float64)
         interp_to_time_float64 = interp_to_times_npdt64.astype(np.float64)
 
         interpolator = scipy.interpolate.interp1d(input_times_float64, values_input, kind='linear')
         result=interpolator(interp_to_time_float64)
+        print(result)
         # Known to fail.  This affects xarray.interp and the current version of tinterpol.
         #self.assertTrue((result >= 0.0).all())
 
+    def test_scipy_spline(self):
+        import scipy
+        import numpy as np
+        time_strings_input = np.array(['2018-07-01T13:02:16.892474880',
+                                       '2018-07-01T13:02:16.922475008',
+                                       '2018-07-01T13:02:16.952474880'])
+        values_input = np.array([0.028584518, 0., 0.013626526],dtype=np.float32)
+
+        input_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_input])
+        interp_to_times_npdt64 = np.array(input_times_npdt64[1])
+
+        input_times_float64 = input_times_npdt64.astype(np.float64)
+        interp_to_time_float64 = interp_to_times_npdt64.astype(np.float64)
+
+        interpolator = scipy.interpolate.make_interp_spline(input_times_float64, values_input, k=1)
+        result=interpolator(interp_to_time_float64)
+        print(result)
+        # make_interp_spline() with k=1 gives the expected result
+        self.assertTrue((result >= 0.0).all())
+
     def test_xarray_interp(self):
         import xarray as xr
         import numpy as np
@@ -334,15 +355,39 @@ def test_xarray_interp(self):
                                        '2018-07-01T13:02:16.922475008',
                                        '2018-07-01T13:02:16.952474880'])
         values_input = np.array([0.028584518, 0., 0.013626526],dtype=np.float32)
-        time_strings_interp_to = np.array(['2018-07-01T13:02:16.922475008'])
 
         input_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_input])
-        interp_to_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_interp_to])
+        interp_to_times_npdt64 = np.array(input_times_npdt64[1])
 
         data_array = xr.DataArray(values_input,dims=['time'],coords={'time':('time',input_times_npdt64)})
 
         result = data_array.interp({"time": interp_to_times_npdt64},method='linear')
         # This is known to fail, due to issues in scipy.interpolate.interp1d
+        print(result.values)
+        # result.values is [-3.469446951953614e-18]
+        #self.assertTrue((result.values >= 0.0).all())
+
+    def test_xarray_interp_float_times(self):
+        import xarray as xr
+        import numpy as np
+
+        time_strings_input = np.array(['2018-07-01T13:02:16.892474880',
+                                       '2018-07-01T13:02:16.922475008',
+                                       '2018-07-01T13:02:16.952474880'])
+        values_input = np.array([0.028584518, 0., 0.013626526],dtype=np.float32)
+
+        input_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_input])
+        interp_to_times_npdt64 = np.array(input_times_npdt64[1])
+
+        input_times_float64 = input_times_npdt64.astype(np.float64)
+        interp_to_time_float64 = interp_to_times_npdt64.astype(np.float64)
+
+        data_array = xr.DataArray(values_input,dims=['time'],coords={'time':('time',input_times_float64)})
+
+        result = data_array.interp({"time": interp_to_time_float64},method='linear')
+        # This is known to fail, due to issues in scipy.interpolate.interp1d
+        print(result.values)
+        # result.values is [-3.469446951953614e-18]
         #self.assertTrue((result.values >= 0.0).all())
 
     def test_numpy_interp(self):

pyspedas/utilities/tests/tplot_wildcard_tests.py

@@ -0,0 +1,129 @@
+"""Test functions in the utilites folder."""
+import unittest
+import pyspedas
+from copy import deepcopy
+
+from pytplot import del_data, tplot, tplot_options, tplot_names
+from pytplot import store_data, get_data
+from pytplot import wildcard_expand, tname_byindex, tplot_wildcard_expand, tindex_byname
+
+global_display=False
+tp_data={'x':[1,2,3], 'y':[4,5,6]}
+
+class UtilTestCases(unittest.TestCase):
+    """Test functions related to tplot variable indexing and wildcards."""
+
+    def test_wildcard_expand(self):
+        # Test wildcard expansion against defined list
+        master_list = ['Foo', 'foo', 'tha_pos_gsm', 'thb_pos_gsm', 'mms1_mec_r_sm']
+        # Literal match
+        result = wildcard_expand(master_list,"Foo")
+        self.assertEqual(result,['Foo'])
+        # Results should be ordered following patterns, not master list
+        result = wildcard_expand(master_list,['foo', 'Foo'])
+        self.assertEqual(result, ['foo', 'Foo'])
+        # Wildcard expansion
+        result = wildcard_expand(master_list,"th?_*_*")
+        self.assertEqual(result,['tha_pos_gsm', 'thb_pos_gsm'])
+        # Case sensitive (by default), no matches found
+        result = wildcard_expand(master_list,"*SM")
+        self.assertEqual(result, [])
+        # Case insensitive
+        result = wildcard_expand(master_list,"*SM", case_sensitive=False)
+        self.assertEqual(result, ['tha_pos_gsm', 'thb_pos_gsm', 'mms1_mec_r_sm'])
+        # Null pattern
+        result = wildcard_expand(master_list, None)
+        self.assertEqual(result, [])
+        # Master list and pattern both null
+        result = wildcard_expand(None,None)
+        self.assertEqual(result, [])
+        # Master list not a list
+        result = wildcard_expand('foo', 'foo')
+        self.assertEqual(result, [])
+
+    def test_tplot_wildcard_expand(self):
+        # Test wildcard expansion against current tplot variables
+        del_data('*')
+        master_list = ['Foo', 'foo', 'tha_pos_gsm', 'thb_pos_gsm', 'mms1_mec_r_sm']
+        for var in master_list:
+            store_data(var,deepcopy(tp_data))
+        # Add a composite variable
+        store_data('composite',data=['foo', 'tha_pos_gsm', 'thb_pos_gsm'])
+        tplot_names()
+        tn = tplot_names(quiet=True)
+        print(tn)
+        # Literal match
+        result = tplot_wildcard_expand("Foo")
+        self.assertEqual(result,['Foo'])
+        # Results should be ordered following patterns, not master list
+        result = tplot_wildcard_expand(['foo', 'Foo'])
+        self.assertEqual(result, ['foo', 'Foo'])
+        # Wildcard expansion
+        result = tplot_wildcard_expand("th?_*_*")
+        self.assertEqual(result,['tha_pos_gsm', 'thb_pos_gsm'])
+        # Composite variable
+        result = tplot_wildcard_expand('composite')
+        self.assertEqual(result, ['composite'])
+        # Case sensitive (by default), no matches found
+        result = tplot_wildcard_expand("*SM")
+        self.assertEqual(result, [])
+        # Case insensitive
+        result = tplot_wildcard_expand("*SM", case_sensitive=False)
+        self.assertEqual(result, ['tha_pos_gsm', 'thb_pos_gsm', 'mms1_mec_r_sm'])
+        # Mixed strings and integers with repeated values
+        result = tplot_wildcard_expand(['Foo',1,2,2,'tha*'], case_sensitive=False)
+        self.assertEqual(result, ['Foo','foo','tha_pos_gsm'])
+        # Null pattern
+        result = tplot_wildcard_expand(None)
+        self.assertEqual(result, [])
+
+    def test_tindex_byname(self):
+        # Test lookups of tplot variable indices by name
+        del_data('*')
+        store_data('foo',data=deepcopy(tp_data))
+        store_data('bar',data=deepcopy(tp_data))
+        result=tindex_byname('foo')
+        self.assertEqual(result,0)
+        result=tindex_byname('bar')
+        self.assertEqual(result,1)
+        result=tindex_byname('doesnt_exist')
+        self.assertEqual(result,None)
+
+    def test_tname_byindex(self):
+        # Test lookups of tplot variable names by indices
+        del_data('*')
+        # Out of bounds (no tplot variables yet)
+        result=tname_byindex(0)
+        self.assertEqual(result,None)
+        store_data('foo',data=deepcopy(tp_data))
+        store_data('bar',data=deepcopy(tp_data))
+        result=tname_byindex(0)
+        self.assertEqual(result,'foo')
+        result=tname_byindex(1)
+        self.assertEqual(result,'bar')
+        # Index too large
+        result=tname_byindex(2)
+        self.assertEqual(result,None)
+        # Negative index
+        result=tname_byindex(-1)
+        self.assertEqual(result,None)
+
+
+    def test_plotting_wildcards_indices(self):
+        # Test tplot using wildcards
+        del_data('*')
+        pyspedas.themis.state(probe=['a', 'b', 'c', 'd', 'e'])
+        # Wildcard pattern
+        tplot_options('title',"Positions of THEMIS A B C D E")
+        tplot('th?_pos', save_png='tplot_wildcard.png', display=global_display)
+        # List of integer indices
+        indices = [tindex_byname('tha_pos'), tindex_byname('thb_pos'), tindex_byname('thc_pos')]
+        tplot_options('title',"Positions of THEMIS A B C")
+        tplot(indices,save_png='tplot_byindex.png', display=global_display)
+        # List including a bad name
+        tplot_options('title',"Positions of THEMIS A B C D E")
+        tplot(['th?_pos', 'doesnt_exist'], save_png='tplot_list_onebad.png', display=global_display)
+
+
+if __name__ == '__main__':
+    unittest.main()

requirements.txt

@@ -1,7 +1,7 @@
 numpy>=1.20.0
 requests
 cdflib
-pytplot-mpl-temp>=2.2.40
+https://github.com/MAVENSDC/PyTplot/archive/matplotlib-backend.zip
 cdasws>=1.7.24
 netCDF4
 pywavelets

setup.py

@@ -39,7 +39,7 @@
         "pywavelets",
         "astropy",
         "hapiclient>=0.2.2",
-        "pytplot-mpl-temp>=2.2.40",
+        "https://github.com/MAVENSDC/PyTplot/archive/matplotlib-backend.zip",
         "viresclient",
     ],
     python_requires=">=3.9",