revamped tests for data product searches

COVERAGE.md

@@ -36,15 +36,15 @@ pyaurorax/search/conjunctions/classes/criteria_block.py           67      0   10
 pyaurorax/search/conjunctions/classes/search.py                  215      1    99%   366
 pyaurorax/search/conjunctions/swarmaurora/__init__.py             15      0   100%
 pyaurorax/search/conjunctions/swarmaurora/_swarmaurora.py         24      0   100%
-pyaurorax/search/data_products/__init__.py                        33      2    94%   124, 234
-pyaurorax/search/data_products/_data_products.py                 100     27    73%   40-44, 86, 92, 96, 101, 128, 143-145, 154, 169, 177-204, 211-214
+pyaurorax/search/data_products/__init__.py                        33      0   100%
+pyaurorax/search/data_products/_data_products.py                  84      2    98%   128, 145
 pyaurorax/search/data_products/classes/data_product.py            47      1    98%   130
-pyaurorax/search/data_products/classes/search.py                 128     46    64%   33, 142, 145, 156-197, 209, 235, 272, 291-292, 300-301, 371-376
-pyaurorax/search/ephemeris/__init__.py                            28      0   100%
+pyaurorax/search/data_products/classes/search.py                 121      0   100%
+pyaurorax/search/ephemeris/__init__.py                            29      0   100%
 pyaurorax/search/ephemeris/_ephemeris.py                          78      2    97%   142, 151
 pyaurorax/search/ephemeris/classes/ephemeris.py                   42      1    98%   97
 pyaurorax/search/ephemeris/classes/search.py                     118      0   100%
-pyaurorax/search/location.py                                      29      6    79%   52-55, 63-66
+pyaurorax/search/location.py                                      29      0   100%
 pyaurorax/search/metadata/__init__.py                             14      0   100%
 pyaurorax/search/metadata/_metadata.py                            20      5    75%   29-31, 45, 54
 pyaurorax/search/metadata_filters.py                              61     25    59%   63-66, 69, 73-78, 86-89, 135-137, 145, 147, 159-169
@@ -94,7 +94,7 @@ pyaurorax/tools/mosaic/_prep_skymaps.py                          119    112
 pyaurorax/tools/spectra/__init__.py                               10      1    90%   129
 pyaurorax/tools/spectra/_plot.py                                 114    108     5%   42-233
 --------------------------------------------------------------------------------------------
-TOTAL                                                           6045   2516    58%
+TOTAL                                                           6023   2437    60%
 
 13 empty files skipped.
 ```

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ Version 1.13.0
   - improve handling of conjunction/ephemeris/data product searches that return no results when retrieving data
   - improve handling of conjunction/ephemeris/data product searches which receive incorrect criteria blocks
   - issue when ordering by 'owner' for `aurorax.search.sources.list()` and `aurorax.search.sources.search()` functions
+  - incorrect typing for `aurorax.search.data_products.search()` function's `data_product_type` parameter (corrected to be list of literals, instead of single literal)
 - removed `aurorax-cli search sources get_stats` command (`aurorax-cli search sources get --include-stats` flag covers this functionality)
 - removed `aurorax-cli search util ground_to_nbtrace` and `aurorax-cli search util ground_to_sbtrace` commands (use library functions instead)
 - removed setter class for `api_headers` and `srs_obj` in `PyAurorax()` objects

pyaurorax/search/data_products/__init__.py

@@ -25,7 +25,6 @@
 from .classes.search import DataProductSearch
 from ..sources.classes.data_source import DataSource
 from ..metadata_filters import MetadataFilter
-from ..._util import show_warning
 from ._data_products import search as func_search
 from ._data_products import upload as func_upload
 from ._data_products import delete as func_delete
@@ -54,7 +53,7 @@ def search(self,
                programs: Optional[List[str]] = None,
                platforms: Optional[List[str]] = None,
                instrument_types: Optional[List[str]] = None,
-               data_product_types: Optional[Literal["keogram", "montage", "movie", "summary_plot", "data_availability"]] = None,
+               data_product_types: Optional[List[Literal["keogram", "montage", "movie", "summary_plot", "data_availability"]]] = None,
                metadata_filters: Optional[Union[MetadataFilter, List[Dict]]] = None,
                metadata_filters_logical_operator: Optional[Literal["and", "or", "AND", "OR"]] = None,
                response_format: Optional[Dict] = None,
@@ -118,13 +117,6 @@ def search(self,
         Returns:
             A `pyaurorax.search.DataProductSearch` object
         """
-        # show warnings
-        if (isinstance(metadata_filters, MetadataFilter) and metadata_filters_logical_operator is not None):
-            # logical operator supplied, but MetadataFilter supplied too
-            show_warning("Supplying a MetadataFilter object in addition to the metadata_filters_logical_operator " +
-                         "parameter is redundant. Only the MetadataFilter object is needed. The " +
-                         "metadata_filters_logical_operator parameter will be ignored")
-
         # return
         return func_search(
             self.__aurorax_obj,
@@ -266,3 +258,49 @@ def get_request_url(self, request_id: str) -> str:
             The request URL
         """
         return func_get_request_url(self.__aurorax_obj, request_id)
+
+    def create_response_format_template(self, default: bool = False) -> Dict:
+        """
+        Generate a template dictionary that can be used as the response_format parameter
+        in a data products search.
+
+        Args:
+            default (bool): 
+                The default value to set for every parameter that can be returned, defaults
+                to False.
+
+        Returns:
+            A template dictionary for the response format
+        """
+        return {
+            "start": default,
+            "end": default,
+            "data_source": {
+                "identifier": default,
+                "program": default,
+                "platform": default,
+                "instrument_type": default,
+                "source_type": default,
+                "display_name": default,
+                "ephemeris_metadata_schema": {
+                    "field_name": default,
+                    "description": default,
+                    "data_type": default,
+                    "allowed_values": default,
+                    "additional_description": default
+                },
+                "data_product_metadata_schema": {
+                    "field_name": default,
+                    "description": default,
+                    "data_type": default,
+                    "allowed_values": default,
+                    "additional_description": default
+                },
+                "owner": default,
+                "maintainers": default,
+                "metadata": default
+            },
+            "url": default,
+            "data_product_type": default,
+            "metadata": default
+        }

pyaurorax/search/data_products/_data_products.py

@@ -37,7 +37,7 @@ def __validate_data_source(aurorax_obj, identifier, records):
     # get data source
     try:
         ds = get_using_identifier(aurorax_obj, identifier, FORMAT_DEFAULT, False)
-    except AuroraXAPIError as e:
+    except AuroraXAPIError as e:  # pragma: nocover
         if ("no data source record found" in str(e).lower()):
             raise AuroraXAPIError("Data source with identifier %d could not be found" % (identifier)) from e
         else:
@@ -87,7 +87,7 @@ def search(aurorax_obj, start, end, programs, platforms, instrument_types, data_
     s.wait(poll_interval=poll_interval, verbose=verbose)
 
     # check if error condition encountered
-    if (s.status["search_result"]["error_condition"] is True):
+    if (s.status["search_result"]["error_condition"] is True):  # pragma: nocover
         # error encountered
         raise AuroraXSearchError(s.logs[-1]["summary"])
 
@@ -140,7 +140,7 @@ def upload(aurorax_obj, identifier, all_records, validate_source, chunk_size):
 
         # evaluate response
         if (res.status_code == 400):
-            if isinstance(res.data, list):
+            if isinstance(res.data, list):  # pragma: nocover
                 raise AuroraXUploadError("%s - %s" % (res.status_code, res.data[0]["message"]))
             raise AuroraXUploadError("%s - %s" % (res.data["error_code"], res.data["error_message"]))
 
@@ -150,7 +150,7 @@ def upload(aurorax_obj, identifier, all_records, validate_source, chunk_size):
 
 def delete_urls(aurorax_obj, data_source, urls):
     # check to make sure the identifier, program, platform, and instrument type are all set in the data source
-    if not all([data_source.identifier, data_source.program, data_source.platform, data_source.instrument_type]):
+    if not all([data_source.identifier, data_source.program, data_source.platform, data_source.instrument_type]):  # pragma: nocover
         raise AuroraXError("One or more required data source parameters are missing, delete operation aborted")
 
     # do request
@@ -165,7 +165,7 @@ def delete_urls(aurorax_obj, data_source, urls):
     res = delete_req.execute()
 
     # evaluate response
-    if (res.status_code == 400):
+    if (res.status_code == 400):  # pragma: nocover
         raise AuroraXAPIError("%s - %s" % (res.data["error_code"], res.data["error_message"]))
 
     # return
@@ -174,7 +174,7 @@ def delete_urls(aurorax_obj, data_source, urls):
 
 def delete(aurorax_obj, data_source, start, end, data_product_types):
     # check to make sure the identifier, program, platform, and instrument type are all set in the data source
-    if not all([data_source.identifier, data_source.program, data_source.platform, data_source.instrument_type]):
+    if not all([data_source.identifier, data_source.program, data_source.platform, data_source.instrument_type]):  # pragma: nocover
         raise AuroraXError("One or more required data source parameters are missing, delete operation aborted")
 
     # do request to get all data products between start and end datetimes
@@ -192,7 +192,7 @@ def delete(aurorax_obj, data_source, start, end, data_product_types):
                    poll_interval=__STANDARD_POLLING_SLEEP_TIME,
                    return_immediately=False,
                    verbose=False)
-    except Exception as e:
+    except Exception as e:  # pragma: nocover
         raise AuroraXError(e) from e
 
     # collect URLs from search result

pyaurorax/search/data_products/classes/data_product.py

@@ -128,8 +128,6 @@ def to_json_serializable(self) -> Dict:
             for key, value in self.metadata.items():
                 if (isinstance(value, datetime.datetime) is True or isinstance(value, datetime.date) is True):
                     self.metadata[key] = self.metadata[key].strftime("%Y-%m-%dT%H:%M:%S.%f")
-        # if (isinstance(self.metadata, list) is True):
-        #     self.metadata = {}
 
         # format data source fields for query
         d["program"] = self.data_source.program

pyaurorax/search/data_products/classes/search.py

@@ -29,8 +29,9 @@
     get_data as requests_get_data,
     get_status as requests_get_status,
 )
+from ...._util import show_warning
 if TYPE_CHECKING:
-    from ....pyaurorax import PyAuroraX
+    from ....pyaurorax import PyAuroraX  # pragma: nocover
 
 
 class DataProductSearch:
@@ -109,11 +110,18 @@ def __init__(self,
                  programs: Optional[List[str]] = None,
                  platforms: Optional[List[str]] = None,
                  instrument_types: Optional[List[str]] = None,
-                 data_product_types: Optional[Literal["keogram", "montage", "movie", "summary_plot", "data_availability"]] = None,
+                 data_product_types: Optional[List[Literal["keogram", "montage", "movie", "summary_plot", "data_availability"]]] = None,
                  metadata_filters: Optional[Union[MetadataFilter, List[Dict]]] = None,
                  metadata_filters_logical_operator: Optional[Literal["and", "or", "AND", "OR"]] = None,
                  response_format: Optional[Dict] = None) -> None:
 
+        # show warnings
+        if (isinstance(metadata_filters, MetadataFilter) and metadata_filters_logical_operator is not None):
+            # logical operator supplied, but MetadataFilter supplied too
+            show_warning("Supplying a MetadataFilter object in addition to the metadata_filters_logical_operator " +
+                         "parameter is redundant. Only the MetadataFilter object is needed. The " +
+                         "metadata_filters_logical_operator parameter will be ignored")
+
         # set variables using passed in args
         self.__aurorax_obj = aurorax_obj
         self.start = start
@@ -174,9 +182,7 @@ def pretty_print(self):
 
         # set logs string
         if (self.executed is True):
-            if (len(self.logs) == 0):
-                logs_str = "[0 log messages]"
-            elif (len(self.logs) == 1):
+            if (len(self.logs) == 1):  # pragma: nocover
                 logs_str = "[1 log message]"
             else:
                 logs_str = "[%d log messages]" % (len(self.logs))
@@ -230,10 +236,6 @@ def query(self):
         # return
         return self.__query
 
-    @query.setter
-    def query(self, query):
-        self.__query = query
-
     def execute(self) -> None:
         """
         Initiate a data product search request
@@ -268,7 +270,7 @@ def update_status(self, status: Optional[Dict] = None) -> None:
             status = requests_get_status(self.__aurorax_obj, self.request_url)
 
         # check response
-        if (status is None):
+        if (status is None):  # pragma: nocover
             raise AuroraXAPIError("Could not retrieve status for this request")
 
         # update request status by checking if data URI is set

pyaurorax/search/ephemeris/__init__.py

@@ -214,3 +214,62 @@ def get_request_url(self, request_id: str) -> str:
             The request URL
         """
         return func_get_request_url(self.__aurorax_obj, request_id)
+
+    def create_response_format_template(self, default: bool = False) -> Dict:
+        """
+        Generate a template dictionary that can be used as the response_format parameter
+        in an ephemeris search.
+
+        Args:
+            default (bool): 
+                The default value to set for every parameter that can be returned, defaults
+                to False.
+
+        Returns:
+            A template dictionary for the response format
+        """
+        return {
+            "data_source": {
+                "identifier": default,
+                "program": default,
+                "platform": default,
+                "instrument_type": default,
+                "source_type": default,
+                "display_name": default,
+                "ephemeris_metadata_schema": {
+                    "field_name": default,
+                    "description": default,
+                    "data_type": default,
+                    "allowed_values": default,
+                    "additional_description": default
+                },
+                "data_product_metadata_schema": {
+                    "field_name": default,
+                    "description": default,
+                    "data_type": default,
+                    "allowed_values": default,
+                    "additional_description": default
+                },
+                "owner": default,
+                "maintainers": default,
+                "metadata": default
+            },
+            "epoch": default,
+            "location_geo": {
+                "lat": default,
+                "lon": default
+            },
+            "location_gsm": {
+                "lat": default,
+                "lon": default
+            },
+            "nbtrace": {
+                "lat": default,
+                "lon": default
+            },
+            "sbtrace": {
+                "lat": default,
+                "lon": default
+            },
+            "metadata": default
+        }

tests/test_suite/cli/search/conjunctions/test_get_data.py

@@ -27,9 +27,9 @@ def test_help(cli_runner):
 
 
 @pytest.mark.cli
-def test_simple(cli_runner, conjunction_search_id):
+def test_simple(cli_runner, api_url, conjunction_search_id):
     # get the data
-    result = cli_runner.invoke(cli, "search conjunctions get_data %s" % (conjunction_search_id))
+    result = cli_runner.invoke(cli, "--api-base-url=%s search conjunctions get_data %s" % (api_url, conjunction_search_id))
     assert result.exit_code == 0
     assert "Checking request status" in result.stdout
 
@@ -40,10 +40,14 @@ def test_simple(cli_runner, conjunction_search_id):
 
 
 @pytest.mark.cli
-def test_with_outfile(cli_runner, conjunction_search_id):
+def test_with_outfile(cli_runner, api_url, conjunction_search_id):
     # get the data
     output_filename = "/tmp/pyaurorax_testing_%s_data.json" % (''.join(random.choices(string.ascii_lowercase + string.digits, k=8)))
-    result = cli_runner.invoke(cli, "search conjunctions get_data %s --outfile=%s" % (conjunction_search_id, output_filename))
+    result = cli_runner.invoke(cli, "--api-base-url=%s search conjunctions get_data %s --outfile=%s" % (
+        api_url,
+        conjunction_search_id,
+        output_filename,
+    ))
     assert result.exit_code == 0
     assert "Checking request status" in result.stdout
 
@@ -53,10 +57,15 @@ def test_with_outfile(cli_runner, conjunction_search_id):
 
 
 @pytest.mark.cli
-def test_with_outfile_indent(cli_runner, conjunction_search_id):
+def test_with_outfile_indent(cli_runner, api_url, conjunction_search_id):
     # get the data
     output_filename = "/tmp/pyaurorax_testing_%s_data.json" % (''.join(random.choices(string.ascii_lowercase + string.digits, k=8)))
-    result = cli_runner.invoke(cli, "search conjunctions get_data %s --outfile=%s --indent=2" % (conjunction_search_id, output_filename))
+    result = cli_runner.invoke(
+        cli, "--api-base-url=%s search conjunctions get_data %s --outfile=%s --indent=2" % (
+            api_url,
+            conjunction_search_id,
+            output_filename,
+        ))
     assert result.exit_code == 0
     assert "Checking request status" in result.stdout
 
@@ -66,10 +75,15 @@ def test_with_outfile_indent(cli_runner, conjunction_search_id):
 
 
 @pytest.mark.cli
-def test_with_outfile_minify(cli_runner, conjunction_search_id):
+def test_with_outfile_minify(cli_runner, api_url, conjunction_search_id):
     # get the data
     output_filename = "/tmp/pyaurorax_testing_%s_data.json" % (''.join(random.choices(string.ascii_lowercase + string.digits, k=8)))
-    result = cli_runner.invoke(cli, "search conjunctions get_data %s --outfile=%s --minify" % (conjunction_search_id, output_filename))
+    result = cli_runner.invoke(
+        cli, "--api-base-url=%s search conjunctions get_data %s --outfile=%s --minify" % (
+            api_url,
+            conjunction_search_id,
+            output_filename,
+        ))
     assert result.exit_code == 0
     assert "Checking request status" in result.stdout
 
@@ -80,8 +94,13 @@ def test_with_outfile_minify(cli_runner, conjunction_search_id):
 
 @pytest.mark.cli
 @pytest.mark.parametrize("arg_value", ["dict", "objects"])
-def test_output_to_terminal(cli_runner, conjunction_search_id, arg_value):
+def test_output_to_terminal(cli_runner, api_url, conjunction_search_id, arg_value):
     # get the data
-    result = cli_runner.invoke(cli, "search conjunctions get_data %s --output-to-terminal=%s" % (conjunction_search_id, arg_value))
+    result = cli_runner.invoke(
+        cli, "--api-base-url=%s search conjunctions get_data %s --output-to-terminal=%s" % (
+            api_url,
+            conjunction_search_id,
+            arg_value,
+        ))
     assert result.exit_code == 0
     assert "Checking request status" in result.stdout