From c69023be905c76ba8ad46c778ea7c65118e74ec7 Mon Sep 17 00:00:00 2001 From: jameswilburlewis Date: Wed, 28 Aug 2024 13:40:05 -0700 Subject: [PATCH 1/4] Temporarily switch to bleeding edge pytplot to test changes to wildcard and indexing logic --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 23f4907c..39d23f6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy>=1.20.0 requests cdflib -pytplot-mpl-temp>=2.2.40 +https://github.com/MAVENSDC/PyTplot/archive/matplotlib-backend.zip cdasws>=1.7.24 netCDF4 pywavelets diff --git a/setup.py b/setup.py index d959b659..6919ce0d 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ "pywavelets", "astropy", "hapiclient>=0.2.2", - "pytplot-mpl-temp>=2.2.40", + "https://github.com/MAVENSDC/PyTplot/archive/matplotlib-backend.zip", "viresclient", ], python_requires=">=3.9", From f230b9e62031fd46d75f9459cac07c2c3551de9f Mon Sep 17 00:00:00 2001 From: jameswilburlewis Date: Wed, 28 Aug 2024 18:41:06 -0700 Subject: [PATCH 2/4] Added/adjusted interpolation tests --- pyspedas/analysis/tests/tests.py | 53 +++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/pyspedas/analysis/tests/tests.py b/pyspedas/analysis/tests/tests.py index c6e2a38d..3b6d4c4d 100644 --- a/pyspedas/analysis/tests/tests.py +++ b/pyspedas/analysis/tests/tests.py @@ -309,23 +309,44 @@ def test_tinterpol(self): def test_scipy_interp1d(self): import scipy + import numpy as np time_strings_input = np.array(['2018-07-01T13:02:16.892474880', '2018-07-01T13:02:16.922475008', '2018-07-01T13:02:16.952474880']) values_input = np.array([0.028584518, 0., 0.013626526],dtype=np.float32) - time_strings_interp_to = np.array(['2018-07-01T13:02:16.922475008']) input_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_input]) - interp_to_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_interp_to]) + interp_to_times_npdt64 = np.array(input_times_npdt64[1]) input_times_float64 = input_times_npdt64.astype(np.float64) interp_to_time_float64 = interp_to_times_npdt64.astype(np.float64) interpolator = scipy.interpolate.interp1d(input_times_float64, values_input, kind='linear') result=interpolator(interp_to_time_float64) + print(result) # Known to fail. This affects xarray.interp and the current version of tinterpol. #self.assertTrue((result >= 0.0).all()) + def test_scipy_spline(self): + import scipy + import numpy as np + time_strings_input = np.array(['2018-07-01T13:02:16.892474880', + '2018-07-01T13:02:16.922475008', + '2018-07-01T13:02:16.952474880']) + values_input = np.array([0.028584518, 0., 0.013626526],dtype=np.float32) + + input_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_input]) + interp_to_times_npdt64 = np.array(input_times_npdt64[1]) + + input_times_float64 = input_times_npdt64.astype(np.float64) + interp_to_time_float64 = interp_to_times_npdt64.astype(np.float64) + + interpolator = scipy.interpolate.make_interp_spline(input_times_float64, values_input, k=1) + result=interpolator(interp_to_time_float64) + print(result) + # make_interp_spline() with k=1 gives the expected result + self.assertTrue((result >= 0.0).all()) + def test_xarray_interp(self): import xarray as xr import numpy as np @@ -334,15 +355,39 @@ def test_xarray_interp(self): '2018-07-01T13:02:16.922475008', '2018-07-01T13:02:16.952474880']) values_input = np.array([0.028584518, 0., 0.013626526],dtype=np.float32) - time_strings_interp_to = np.array(['2018-07-01T13:02:16.922475008']) input_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_input]) - interp_to_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_interp_to]) + interp_to_times_npdt64 = np.array(input_times_npdt64[1]) data_array = xr.DataArray(values_input,dims=['time'],coords={'time':('time',input_times_npdt64)}) result = data_array.interp({"time": interp_to_times_npdt64},method='linear') # This is known to fail, due to issues in scipy.interpolate.interp1d + print(result.values) + # result.values is [-3.469446951953614e-18] + #self.assertTrue((result.values >= 0.0).all()) + + def test_xarray_interp_float_times(self): + import xarray as xr + import numpy as np + + time_strings_input = np.array(['2018-07-01T13:02:16.892474880', + '2018-07-01T13:02:16.922475008', + '2018-07-01T13:02:16.952474880']) + values_input = np.array([0.028584518, 0., 0.013626526],dtype=np.float32) + + input_times_npdt64 = np.array([np.datetime64(t) for t in time_strings_input]) + interp_to_times_npdt64 = np.array(input_times_npdt64[1]) + + input_times_float64 = input_times_npdt64.astype(np.float64) + interp_to_time_float64 = interp_to_times_npdt64.astype(np.float64) + + data_array = xr.DataArray(values_input,dims=['time'],coords={'time':('time',input_times_float64)}) + + result = data_array.interp({"time": interp_to_time_float64},method='linear') + # This is known to fail, due to issues in scipy.interpolate.interp1d + print(result.values) + # result.values is [-3.469446951953614e-18] #self.assertTrue((result.values >= 0.0).all()) def test_numpy_interp(self): From 5b357da98026ef2bb0bd18c3a5b14c6fd7bd27d0 Mon Sep 17 00:00:00 2001 From: jameswilburlewis Date: Wed, 28 Aug 2024 18:43:39 -0700 Subject: [PATCH 3/4] Add test suite for tplot wildcard expansion and related tools --- .github/workflows/pythonpackage.yml | 2 + .../utilities/tests/tplot_wildcard_tests.py | 129 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 pyspedas/utilities/tests/tplot_wildcard_tests.py diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4da7a707..d5fb968e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -239,6 +239,8 @@ jobs: coverage run -a -m pyspedas.utilities.tests.download_tests echo Starting utilities misc tests at `date` coverage run -a -m pyspedas.utilities.tests.misc_tests + echo Starting tplot wildcard tests at `date` + coverage run -a -m pyspedas.utilities.tests.tplot_wildcard_tests echo Starting plotting tests at `date` coverage run -a -m pyspedas.utilities.tests.plot_tests echo Starting utilities time_tests tests at `date` diff --git a/pyspedas/utilities/tests/tplot_wildcard_tests.py b/pyspedas/utilities/tests/tplot_wildcard_tests.py new file mode 100644 index 00000000..9b43c123 --- /dev/null +++ b/pyspedas/utilities/tests/tplot_wildcard_tests.py @@ -0,0 +1,129 @@ +"""Test functions in the utilites folder.""" +import unittest +import pyspedas +from copy import deepcopy + +from pytplot import del_data, tplot, tplot_options, tplot_names +from pytplot import store_data, get_data +from pytplot import wildcard_expand, tname_byindex, tplot_wildcard_expand, tindex_byname + +global_display=False +tp_data={'x':[1,2,3], 'y':[4,5,6]} + +class UtilTestCases(unittest.TestCase): + """Test functions related to tplot variable indexing and wildcards.""" + + def test_wildcard_expand(self): + # Test wildcard expansion against defined list + master_list = ['Foo', 'foo', 'tha_pos_gsm', 'thb_pos_gsm', 'mms1_mec_r_sm'] + # Literal match + result = wildcard_expand(master_list,"Foo") + self.assertEqual(result,['Foo']) + # Results should be ordered following patterns, not master list + result = wildcard_expand(master_list,['foo', 'Foo']) + self.assertEqual(result, ['foo', 'Foo']) + # Wildcard expansion + result = wildcard_expand(master_list,"th?_*_*") + self.assertEqual(result,['tha_pos_gsm', 'thb_pos_gsm']) + # Case sensitive (by default), no matches found + result = wildcard_expand(master_list,"*SM") + self.assertEqual(result, []) + # Case insensitive + result = wildcard_expand(master_list,"*SM", case_sensitive=False) + self.assertEqual(result, ['tha_pos_gsm', 'thb_pos_gsm', 'mms1_mec_r_sm']) + # Null pattern + result = wildcard_expand(master_list, None) + self.assertEqual(result, []) + # Master list and pattern both null + result = wildcard_expand(None,None) + self.assertEqual(result, []) + # Master list not a list + result = wildcard_expand('foo', 'foo') + self.assertEqual(result, []) + + def test_tplot_wildcard_expand(self): + # Test wildcard expansion against current tplot variables + del_data('*') + master_list = ['Foo', 'foo', 'tha_pos_gsm', 'thb_pos_gsm', 'mms1_mec_r_sm'] + for var in master_list: + store_data(var,deepcopy(tp_data)) + # Add a composite variable + store_data('composite',data=['foo', 'tha_pos_gsm', 'thb_pos_gsm']) + tplot_names() + tn = tplot_names(quiet=True) + print(tn) + # Literal match + result = tplot_wildcard_expand("Foo") + self.assertEqual(result,['Foo']) + # Results should be ordered following patterns, not master list + result = tplot_wildcard_expand(['foo', 'Foo']) + self.assertEqual(result, ['foo', 'Foo']) + # Wildcard expansion + result = tplot_wildcard_expand("th?_*_*") + self.assertEqual(result,['tha_pos_gsm', 'thb_pos_gsm']) + # Composite variable + result = tplot_wildcard_expand('composite') + self.assertEqual(result, ['composite']) + # Case sensitive (by default), no matches found + result = tplot_wildcard_expand("*SM") + self.assertEqual(result, []) + # Case insensitive + result = tplot_wildcard_expand("*SM", case_sensitive=False) + self.assertEqual(result, ['tha_pos_gsm', 'thb_pos_gsm', 'mms1_mec_r_sm']) + # Mixed strings and integers with repeated values + result = tplot_wildcard_expand(['Foo',1,2,2,'tha*'], case_sensitive=False) + self.assertEqual(result, ['Foo','foo','tha_pos_gsm']) + # Null pattern + result = tplot_wildcard_expand(None) + self.assertEqual(result, []) + + def test_tindex_byname(self): + # Test lookups of tplot variable indices by name + del_data('*') + store_data('foo',data=deepcopy(tp_data)) + store_data('bar',data=deepcopy(tp_data)) + result=tindex_byname('foo') + self.assertEqual(result,0) + result=tindex_byname('bar') + self.assertEqual(result,1) + result=tindex_byname('doesnt_exist') + self.assertEqual(result,None) + + def test_tname_byindex(self): + # Test lookups of tplot variable names by indices + del_data('*') + # Out of bounds (no tplot variables yet) + result=tname_byindex(0) + self.assertEqual(result,None) + store_data('foo',data=deepcopy(tp_data)) + store_data('bar',data=deepcopy(tp_data)) + result=tname_byindex(0) + self.assertEqual(result,'foo') + result=tname_byindex(1) + self.assertEqual(result,'bar') + # Index too large + result=tname_byindex(2) + self.assertEqual(result,None) + # Negative index + result=tname_byindex(-1) + self.assertEqual(result,None) + + + def test_plotting_wildcards_indices(self): + # Test tplot using wildcards + del_data('*') + pyspedas.themis.state(probe=['a', 'b', 'c', 'd', 'e']) + # Wildcard pattern + tplot_options('title',"Positions of THEMIS A B C D E") + tplot('th?_pos', save_png='tplot_wildcard.png', display=global_display) + # List of integer indices + indices = [tindex_byname('tha_pos'), tindex_byname('thb_pos'), tindex_byname('thc_pos')] + tplot_options('title',"Positions of THEMIS A B C") + tplot(indices,save_png='tplot_byindex.png', display=global_display) + # List including a bad name + tplot_options('title',"Positions of THEMIS A B C D E") + tplot(['th?_pos', 'doesnt_exist'], save_png='tplot_list_onebad.png', display=global_display) + + +if __name__ == '__main__': + unittest.main() From 7018db9e95a52d11d09025dd9b468b47bdc953cc Mon Sep 17 00:00:00 2001 From: jameswilburlewis Date: Wed, 28 Aug 2024 21:23:16 -0700 Subject: [PATCH 4/4] Disabling VIRES tests until we figure out why we're getting flooded with HTTP 200 responses --- .github/workflows/pythonpackage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index d5fb968e..7bcbc5f3 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -142,8 +142,8 @@ jobs: echo "Disk space after cleanup:" df -h . echo "=========================================================" - echo Starting VIRES client tests at `date` - coverage run -a -m pyspedas.vires.tests.tests + #echo Starting VIRES client tests at `date` + #coverage run -a -m pyspedas.vires.tests.tests echo Starting secs tests at `date` coverage run -a -m pyspedas.secs.tests.tests echo Starting MTH5 import test at `date`