From f6a768febbcd622b396e60640ca5eacbe246ff08 Mon Sep 17 00:00:00 2001 From: Wolfgang Preimesberger Date: Thu, 7 Sep 2023 11:22:13 +0200 Subject: [PATCH] Add method to export geojson from metadata (#71) * Add new networks * Update map plot * Add option to export network collection metadata to geojson * Update tests * Fix docs * Update docs env name * Update docs * Update changelog --- .readthedocs.yml | 2 +- CHANGELOG.rst | 4 ++- README.rst | 9 ++--- docs/{docs_env.yml => env.yml} | 14 +++++--- docs/examples/index.rst | 11 ------ docs/index.rst | 5 +-- src/ismn/citations.txt | 8 ++++- src/ismn/components.py | 66 ++++++++++++++++++++++++++++++++++ src/ismn/interface.py | 61 +++++++++++++++++++++---------- tests/test_components.py | 15 ++++++-- tests/test_filecollection.py | 4 --- 11 files changed, 149 insertions(+), 50 deletions(-) rename docs/{docs_env.yml => env.yml} (53%) delete mode 100644 docs/examples/index.rst diff --git a/.readthedocs.yml b/.readthedocs.yml index 10d2329..d7cb498 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -14,4 +14,4 @@ sphinx: configuration: docs/conf.py conda: - environment: docs/docs_env.yml + environment: docs/env.yml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 16b51b8..7f5ea00 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,9 @@ Changelog Unreleased changes in master branch =================================== -- +- Added method to NetworkCollection to export metadata as (geo)json. +- Added more options when plotting the station overview map. +- Network citation list updated. Version 1.3.4 ============= diff --git a/README.rst b/README.rst index a8d5c92..824abee 100644 --- a/README.rst +++ b/README.rst @@ -23,12 +23,13 @@ The full documentation is available at https://ismn.readthedocs.io and includes a tutorial on reading ISMN data in python after downloading it from https://ismn.earth +The following **tutorials** are also available as ipython notebooks ``docs/examples``: -The following tutorials are available in ``docs/examples``: + #. `ISMN reader basic functionality `_ + #. `Adding custom metadata readers `_ - `1) ISMN reader basic functionality `_ - - `2) Adding custom metadata readers `_ +Data used in the tutorials is *not* provided in this package. Please create an account at `ismn.earth `_ +to download the required files. Citation ======== diff --git a/docs/docs_env.yml b/docs/env.yml similarity index 53% rename from docs/docs_env.yml rename to docs/env.yml index dd83d97..4f43d5c 100644 --- a/docs/docs_env.yml +++ b/docs/env.yml @@ -1,11 +1,17 @@ # To keep the RTD build as small as possible, we define a separate .yml here -name: ismn_docs +name: docs channels: - conda-forge - defaults dependencies: -- python=3.8 +- python=3.10 - ipykernel - nbsphinx -# Note: RTD will add packages it needs automatically such as sphinx, sphinx_rtd_theme - +- mock +- pillow +- sphinx<7 +- sphinx_rtd_theme +- pip +- pip: + - recommonmark + - readthedocs-sphinx-ext diff --git a/docs/examples/index.rst b/docs/examples/index.rst deleted file mode 100644 index 80fb061..0000000 --- a/docs/examples/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -Examples -******** - -.. toctree:: - :maxdepth: 1 - - interface - custom_meta - -All shown examples are also available as ipython notebooks in ``ismn/docs/examples``. -Data used in the examples is NOT provided here, and must be downloaded from `ISMN `_. diff --git a/docs/index.rst b/docs/index.rst index d77e1df..90c810a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,8 @@ :glob: README - Examples + Tutorial 1: Reading ISMN Data + Tutorial 2: Add and use custom metadata License Authors Changelog @@ -18,4 +19,4 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` -* :ref:`search` \ No newline at end of file +* :ref:`search` diff --git a/src/ismn/citations.txt b/src/ismn/citations.txt index 53de092..60fc85c 100644 --- a/src/ismn/citations.txt +++ b/src/ismn/citations.txt @@ -10,7 +10,7 @@ AMMA-CATCH;Lebel, Thierry, Cappelaere, Bernard, Galle, Sylvie, Hanan, Niall, Ker ARM;Cook, D. R. (2016a), Soil temperature and moisture profile (stamp) system handbook, Technical report, DOE Office of Science Atmospheric Radiation Measurement (ARM) Program. ARM;Cook, D. R. (2016b), Soil water and temperature system (swats) instrument handbook, Technical report, DOE Office of Science Atmospheric Radiation Measurement (ARM) Program. ARM;Cook, D. R. & Sullivan, R. C. (2018), Surface energy balance system (sebs) instrument handbook, Technical report, DOE Office of Science Atmospheric Radiation Measurement (ARM) Program. -AWDS;We acknowledge the work of Natalie Umphlett (http://www.hprcc.unl.edu/awdn.php) in support of the ISMN. +AWDN;We acknowledge the work of Natalie Umphlett (http://www.hprcc.unl.edu/awdn.php) in support of the ISMN. BIEBRZA_S-1;Musial, J. P., Dabrowska-Zielinska, K., Kiryla, W., Oleszczuk, R., Gnatowski, T. & Jaszczynski, J. (2016), ‘Derivation and validation of the high resolution satellite soil moisture products: a case study of the biebrza sentinel-1 validation sites’, Geoinformation Issues 8(1 (8)), 37–53. BNZ-LTER;Van Cleve, Keith, Chapin, F.S. Stuart, Ruess, Roger W. 2015. Bonanza Creek Long Term Ecological Research Project Climate Database - University of Alaska Fairbanks. BNZ-LTER;Bonanza Creek Long Term Ecological Research Project Climate Database. 2015. University of Alaska Fairbanks. @@ -52,6 +52,7 @@ IMA_CAN1;Raffelli, G., Id, M., Previati, M., Id, D., Canone, D., Gisolo, D., Bev IOWA;Robock, Alan, Konstantin Y. Vinnikov, Govindarajalu Srinivasan, Jared K. Entin, Steven E. Hollinger, Nina A. Speranskaya, Suxia Liu, and A. Namkhai, 2000: The Global Soil Moisture Data Bank. Bull. Amer. Meteorol. Soc., 81, 1281-1299 IPE;Alday, J. G., Camarero, J. J., Revilla, J. & Dios, V. R. (2020), ‘Similar diurnal, seasonal and annual rhythms in radial root expansion across two coexisting mediterranean oak species’, Tree Physiology. iRON;Osenga, E. C., Arnott, J. C., Endsley, K. A., & Katzenberger, J. W. (2019). Bioclimatic and soil moisture monitoring across elevation in a mountain watershed: Opportunities for research and resource management. Water Resources Research, 55. https://doi.org/10.1029/2018WR023653 +KHOREZM;We acknowledge the work of Patrick Knöfel and the KHOREZM team in support of the ISMN. KIHS_CMC;We acknowledge the work of Yeon gil Lee, Yongjun Lee, Kiyoung Kim (http://kihs.re.kr) in support of the ISMN. KIHS_SMC;We acknowledge the work of Minha Choi, Yeon gil Lee (http://kihs.re.kr) in support of the ISMN. LAB-net;Mattar, C., Santamaria-Artigas, A., Duran-Alarcon, C., Olivera-Guerra, L., Fuster, R. & Borvar´an, D. (2016), ‘The lab-net soil moisture network: Application to thermal remote sensing and surface energy balance’, Data 1(1). @@ -65,6 +66,7 @@ MySMNet;Kang, C. S., Kanniah, K. D. & Kerr, Y. H. (2019), ‘Calibration of smos NAQU;Su, Z., Wen, J., Dente, L., Van Der Velde, R., Wang, L., Ma, Y., Yang, K. and Hu, Z., 2011. The Tibetan Plateau observatory of plateau scale soil moisture and soil temperature (Tibet-Obs) for quantifying uncertainties in coarse resolution satellite and model products. Hydrology & Earth System Sciences, 15(7). NGARI;Su, Z., Wen, J., Dente, L., Van Der Velde, R., Wang, L., Ma, Y., Yang, K. and Hu, Z., 2011. The Tibetan Plateau observatory of plateau scale soil moisture and soil temperature (Tibet-Obs) for quantifying uncertainties in coarse resolution satellite and model products. Hydrology & Earth System Sciences, 15(7). NVE;We acknowledge the work of Fred Wenger (https://www.nve.no/hydrology/?ref=mainmenu) in support of the ISMN. +ORACLE;We acknowledge the work of Arnaud Blanchouin and ORACLE team of the Institut national de recherche en sciences et technologies pour l"environment et l"agriculture, France in support of the ISMN OZNET;Smith, A. B., J. P.Walker, A. W.Western, R. I.Young, K. M.Ellett, R. C.Pipunic, R. B.Grayson, L.Siriwardena, F. H. S.Chiew, and H.Richter (2012), The Murrumbidgee soil moisture monitoring network data set, Water Resour. Res., 48, W07701, doi:10.1029/2012WR011976. OZNET;Young, R., Walker, J., Yeoh, N., Smith, A., Ellett, K., Merlin, O. and Western, A., 2008. Soil moisture and meteorological observations from the murrumbidgee catchment. Department of Civil and Environmental Engineering, The University of Melbourne. PBO_H2O;Kristine M. Larson, Eric E. Small, Ethan D. Gutmann, Andria L. Bilich, John J. Braun, Valery U. Zavorotny: Use of GPS receivers as a soil moisture network for water cycle studies. GEOPHYSICAL RESEARCH LETTERS, VOL. 35, L24405, doi:10.1029/2008GL036013, 2008. @@ -91,6 +93,8 @@ SNOTEL;Leavesley, G., David, O., Garen, D., Lea, J., Marron, J., Pagano, T., Per SOILSCAPE;Moghaddam, M., D. Entekhabi, Y. Goykhman, K. Li, M. Liu, A. Mahajan, A. Nayyar, D. Shuman, and D. Teneketzis, A wireless soil moisture smart sensor web using physics-based optimal control: concept and initial demonstration IEEE-JSTARS, , vol. 3, no. 4, pp. 522-535, December 2010 SOILSCAPE;Moghaddam, M., A.R. Silva, D. Clewley, R. Akbar, S.A. Hussaini, J. Whitcomb, R. Devarakonda, R. Shrestha, R.B. Cook, G. Prakash, S.K. Santhana Vannan, and A.G. Boyer. 2016. Soil Moisture Profiles and Temperature Data from SoilSCAPE Sites, USA. ORNL DAAC, Oak Ridge, Tennessee, USA. http://dx.doi.org/10.3334/ORNLDAAC/1339 SOILSCAPE;Shuman, D. I., Nayyar, A., Mahajan, A., Goykhman, Y., Li, K., Liu, M., Teneketzis, D., Moghaddam, M. & Entekhabi, D. (2010), ‘Measurement scheduling for soil moisture sensing: From physical models to optimal control’, Proceedings of the IEEE 98(11), 1918–1933. +SONTE-China;Bingze Li, Chunmei Wang, Xingfa Gu, Xiang Zhou et al. 2022. Accuracy calibration and evaluation of capacitance-based soil moisture sensors for a variety of soil properties. Agricultural Water Management, 273, 107913. https://doi.org/10.1016/j.agwat.2022.107913. +SONTE-China;Leran Han, Chunmei Wang, Tao Yu, Xingfa Gu et al. High-Precision soil moisture mapping based on multi-model coupling and background knowledge, over vegetated areas using Chinese GF-3 and GF-1 satellite data. Remote Sensing, 2020, 12(13):2123. STEMS;Darouich, H., Ramos, T.B., Pereira, L.S., Rabino, D., Bagagiolo, G., Capello, G., Simionesei, L., Cavallo, E., Biddoccu, M. Water Use and Soil Water Balance of Mediterranean Vineyards under Rainfed and Drip Irrigation Management: Evapotranspiration Partition and Soil Management Modelling for Resource Conservation. Water 2022, 14, 554. https://doi.org/10.3390/w14040554 STEMS;Capello G, Biddoccu M, Ferraris S, Cavallo E, 2019. Effects of tractor passes on hydrological and soil erosion processes in tilled and grassed vineyards. Water 2019, 11(10), 2118, https://doi.org/10.3390/w11102118 SWEX_POLAND;W. Marczewski, J. Slominski, E. Slominska, B. Usowicz, J. Usowicz, S. Romanov, O. Maryskevych, J. Nastula, and J. Zawadzki, 2010: Strategies for validating and directions for employing SMOS data, in the Cal-Val project SWEX (3275) for wetlands, Hydrol. Earth Syst. Sci. Discuss., 7, 7007–7057 @@ -101,6 +105,7 @@ TAHMO;We acknowledge the work of Nicolaas Cornelis van de Giesen and Frank Annor TERENO;Zacharias, S., H.R. Bogena, L. Samaniego, M. Mauder, R. Fuß, T. Puetz, M. Frenzel, M. Schwank, C. Baessler, K. Butterbach-Bahl, O. Bens, E. Borg, A. Brauer, P. Dietrich, I. Hajnsek, G. Helle, R. Kiese, H. Kunstmann, S. Klotz, J.C. Munch, H. Papen, E. Priesack, H. P. Schmid, R. Steinbrecher, U. Rosenbaum, G. Teutsch, H. Vereecken. 2011. A Network of Terrestrial Environmental Observatories in Germany. Vadose Zone J. 10. 955–973. doi:10.2136/vzj2010.0139 TERENO;Bogena, H., Kunkel, R., Puetz, T., Vereecken, H., Kruger, E., Zacharias, S., Dietrich, P., Wollschlaeger, U., Kunstmann, H., Papen, H., Schmid, H., Munch, J., Priesack, E., Schwank, M., Bens, O., Brauer, A., Borg, E. & Hajnsek, I. (2012), ‘Tereno - long-term monitoring network for terrestrial environmental research’, Hydrologie und Wasserbewirtschaftung 56, 138–143. TERENO;Bogena, H. R. (2016), ‘Tereno: German network of terrestrial environmental observatories’, Journal of large-scale research facilities JLSRF 2, 52. +TWENTE;Van der Velde, R., Benninga, H. J. F., Retsios, B., Vermunt, P. C., & Salama, M. S. (2023). Twelve years of profile soil moisture and temperature measurements in Twente, the Netherlands. Earth System Science Data, 15(4), 1889-1910. https://doi.org/10.5194/essd-15-1889-2023 TxSON;Caldwell, T. G., T. Bongiovanni, M. H. Cosh, T. J. Jackson, A. Colliander, C. J. Abolt, R. Casteel, T. Larson, B. R. Scanlon, and M. H. Young (2019), The Texas Soil Observation Network: A comprehensive soil moisture dataset for remote sensing and land surface model validation, Vadose Zone Journal, 18:100034, doi:10.2136/vzj2019.04.0034 UDC_SMOS;Schlenz, F., Dall'Amico, J., Loew, A., Mauser, W. (2012): Uncertainty Assessment of the SMOS Validation in the Upper Danube Catchment. IEEE Transactions on Geoscience and Remote Sensing, 50(5), pp.1517–1529. doi: 10.1109/TGRS.2011.2171694. UDC_SMOS;A. Loew, J. T. Dall'Amico, F. Schlenz, W. Mauser (2009): The Upper Danube soil moisture validation site: measurements and activities, paper presented at Earth Observation and Water Cycle conference, Frascati (Rome), 18 - 20 November 2009, to be published in ESA Special dataation SP-674. @@ -112,6 +117,7 @@ USCRN;Bell, J. E., M. A. Palecki, C. B. Baker, W. G. Collins, J. H. Lawrimore, R USDA-ARS;Jackson, T.J., Cosh, M.H., Bindlish, R., Starks, P.J., Bosch, D.D., Seyfried, M.S., Goodrich, D.C., Moran, M.S., Validation of Advanced Microwave Scanning Radiometer Soil Moisture Products. IEEE Transactions on Geoscience and Remote Sensing. 48: 4256-4272, 2010. VAS;We acknowledge the work of Mike Schwank, Jean-Pierre Wigneron, Yann H. Kerr, Diego Intrigliolo, Jennifer Grant, Ernesto Lopez-Baeza (http://nimbus.uv.es/) in support of the ISMN. VDS;We acknowledge the work of Richard de Jeu (https://www.vandersat.com/) in support of the ISMN. +XMS-CAT;We acknowledge the work of Agnès Lladós and Lola Boquera as well as the XMS-CAT network team in support of the ISMN. WEGENERNET;Kirchengast, G., Kabas, T., Leuprecht, A., Bichler, C. & Truhetz, H. (2014), ‘Wegenernet: A pioneering high-resolution network for monitoring weather and climate’, Bulletin of the American Meteorological Society 95. WEGENERNET;Fuchsberger, J., Kirchengast, G. & Kabas, T. (2020), ‘Wegenernet high-resolution weather and climate data 2007 to 2019’, Earth System Science Data Discussions 2020, 1–49. WSMN;Petropoulos, G. P. & McCalmont, J. P. (2017), ‘An operational in situ soil moisture & soil temperature monitoring network for west wales, uk: The wsmn network’, Sensors 17(7), 1481. diff --git a/src/ismn/components.py b/src/ismn/components.py index f872300..5e93674 100644 --- a/src/ismn/components.py +++ b/src/ismn/components.py @@ -31,6 +31,8 @@ from ismn.meta import MetaData, Depth from ismn.const import deprecated, CITATIONS +import json + logger = logging.getLogger(__name__) ch = logging.StreamHandler() @@ -820,3 +822,67 @@ def export_citations(self, out_file=None): out_file.write("\n") return refs + + def export_geojson(self, path, extra_props=None, **filter_kwargs): + """ + Filter sensors in collection and create geojson file containing all + features. + + Parameters + ---------- + path: str + Path to geojson file + extra_props: list[str] + List of extra properties to include in geojson file + By default only depth_from and depth_to are included + e.g. ['timerange_from', 'timerange_to', 'variable', 'frm_class'] + filter_kwargs: + Keyword arguments to filter sensors in collection + """ + extra_props = extra_props or [] + geoinfo = { + "type": "FeatureCollection", + "features": [], + } + + for nw in self.iter_networks(): + feature = { + "type": "Feature", + "geometry": { + "type": "MultiPoint", + "coordinates": [], + "properties": { + "datasetName": nw.name, + "datasetVersion": 1, + "datasetProperties": [], + } + } + } + for station, sensor in nw.iter_sensors(**filter_kwargs): + feature["geometry"]["coordinates"].append([ + station.lon, + station.lat + ]) + + feature["geometry"]["properties"]["datasetProperties"] += [ + { + "propertyName": "depth_from", + "propertyValue": sensor.depth[0] + }, + { + "propertyName": "depth_to", + "propertyValue": sensor.depth[1] + }, + ] + for prop in extra_props: + feature["geometry"]["properties"]["datasetProperties"] += [ + { + "propertyName": prop, + "propertyValue": str(sensor.metadata[prop].val), + } + ] + + geoinfo["features"].append(feature) + + with open(path, 'w') as f: + json.dump(geoinfo, f, ensure_ascii=False) diff --git a/src/ismn/interface.py b/src/ismn/interface.py index 09b73be..63eedb0 100644 --- a/src/ismn/interface.py +++ b/src/ismn/interface.py @@ -620,9 +620,13 @@ def plot_station_locations( variable=None, min_depth=-np.inf, max_depth=np.inf, + extent=None, stats_text=True, check_only_sensor_depth_from=False, - markersize=1, + markersize=12.5, + markeroutline=True, + borders=True, + legend=True, text_scalefactor=1, dpi=300, filename=None, @@ -642,13 +646,23 @@ def plot_station_locations( max_depth : float, optional (default: -np.inf) See description of min_depth. This is the bottom threshold for the allowed depth. + extent: list, optional (default: None) + [lon min, lon max, lat min, lat max] + Extent of the map that is plotted. If None is passed, a global map + is plotted. stats_text : bool, optianal (default: False) Include text of net/stat/sens counts in plot. check_only_sensor_depth_from : bool, optional (default: False) Ignores the sensors depth_to value and only checks if depth_from of the sensor is in the passed depth_range (e.g. for cosmic ray probes). - markersize : int, optional (default: 1) + markersize : int or float, optional (default: 12.5) Size of the marker, might depend on the amount of stations you plot. + markeroutline: bool, optional (default: True) + If True, a black outline is drawn around the markers. + borders: bool, optional (default: True) + If True, country borders are drawn. + legend: bool, optional (default: True) + If True, a legend is drawn. text_scalefactor : float, optional (default: 1) Scale factor that is applied to header and legend. dpi: float, optional (default: 300) @@ -689,12 +703,14 @@ def plot_station_locations( else: fig = None - ax.coastlines(linewidth=0.5) + ax.coastlines() # show global map ax.set_global() - ax.add_feature(cfeature.BORDERS, linewidth=0.5, edgecolor="gray") - if not (sys.version_info[0] == 3 and sys.version_info[1] == 4): + if borders: + ax.add_feature(cfeature.BORDERS, edgecolor="gray") ax.add_feature(cfeature.STATES, linewidth=0.5, edgecolor="gray") + + if not (sys.version_info[0] == 3 and sys.version_info[1] == 4): colormap = plt.get_cmap("tab20") else: colormap = plt.get_cmap("Set1") @@ -722,16 +738,22 @@ def plot_station_locations( if stat.name not in act_stations: act_stations.append(stat.name) - ax.plot( + ax.scatter( stat.lon, stat.lat, color=netcolor, - markersize=markersize, + s=markersize, + linewidth=0.5, marker="s", transform=data_crs, + edgecolors="black" if markeroutline else None, + zorder=2, ) n_sens += 1 + if extent is not None: + ax.set_extent(extent, crs=data_crs) + nrows = 8.0 if len(act_networks) > 8 else len(act_networks) try: @@ -741,18 +763,19 @@ def plot_station_locations( if ncols == 0: ncols = 1 - handles, labels = ax.get_legend_handles_labels() - lgd = ax.legend( - handles, labels, loc="lower center", bbox_to_anchor=(0.5, -0.1)) - - ax.legend( - rect, - act_networks, - loc="upper center", - ncol=ncols, - bbox_to_anchor=(0.5, -0.05), - fontsize=4 * text_scalefactor, - ) + if legend: + handles, labels = ax.get_legend_handles_labels() + lgd = ax.legend( + handles, labels, loc="lower center", bbox_to_anchor=(0.5, -0.1)) + + ax.legend( + rect, + act_networks, + loc="upper center", + ncol=ncols, + bbox_to_anchor=(0.5, -0.05), + fontsize=4 * text_scalefactor, + ) postfix_depth = ("when only considering depth_from of the sensor" if check_only_sensor_depth_from else "") diff --git a/tests/test_components.py b/tests/test_components.py index b594465..84ef255 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -31,6 +31,8 @@ from ismn.components import NetworkCollection, Network, Station, Sensor, Depth from pygeogrids.grids import BasicGrid +from tempfile import TemporaryDirectory +import json rpath = os.path.join(os.path.dirname(__file__), "test_data") @@ -74,6 +76,16 @@ def test_references(self): refs = self.netcol.export_citations(out_file=None) assert len(refs.keys()) == 2 + def test_json_dump(self): + with TemporaryDirectory() as temp: + self.netcol.export_geojson(os.path.join(temp, "meta.json")) + with open(os.path.join(temp, "meta.json")) as f: + meta_dict = json.load(f) + for feature in meta_dict['features']: + net = feature['geometry']['properties']['datasetName'] + assert self.netcol[net][0][0].depth[0] == 0.5 + assert self.netcol[net][0][0].depth[1] == 1.0 + class NetworkTest(unittest.TestCase): def setUp(self): """ @@ -274,6 +286,3 @@ def test_read_data(self): assert data.index.size == 7059 assert data.columns.size == 3 - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_filecollection.py b/tests/test_filecollection.py index 2406f62..1c1aca9 100644 --- a/tests/test_filecollection.py +++ b/tests/test_filecollection.py @@ -160,7 +160,3 @@ def setUpClass(cls): cleanup(metadata_path) cls.coll = IsmnFileCollection.build_from_scratch(testdata_zip_path) - - -if __name__ == "__main__": - unittest.main()