updated test suite for all of search functions

COVERAGE.md

@@ -21,7 +21,7 @@ pyaurorax/data/ucalgary/read/__init__.py                          36      0   10
 pyaurorax/exceptions.py                                           28      0   100%
 pyaurorax/models/__init__.py                                      10      0   100%
 pyaurorax/models/atm/__init__.py                                  17      0   100%
-pyaurorax/pyaurorax.py                                           203      1    99%   180
+pyaurorax/pyaurorax.py                                           206      1    99%   174
 pyaurorax/search/__init__.py                                      93      0   100%
 pyaurorax/search/api/__init__.py                                  20      0   100%
 pyaurorax/search/api/classes/request.py                           33      0   100%
@@ -48,12 +48,12 @@ pyaurorax/search/location.py                                      29      0   10
 pyaurorax/search/metadata/__init__.py                             14      0   100%
 pyaurorax/search/metadata/_metadata.py                            16      0   100%
 pyaurorax/search/metadata_filters.py                              60      0   100%
-pyaurorax/search/requests/__init__.py                             28      2    93%   144, 207
-pyaurorax/search/requests/_requests.py                           100     24    76%   50, 54, 88, 115, 122-125, 129, 136-139, 144, 152, 154, 156, 158, 167, 169, 177-184
-pyaurorax/search/sources/__init__.py                              59     26    56%   183-240
-pyaurorax/search/sources/_sources.py                             119     13    89%   132, 154-155, 208, 211, 213, 215, 217, 245-246, 257, 293-294
-pyaurorax/search/sources/classes/data_source.py                   68     26    62%   170, 173, 186-212
-pyaurorax/search/sources/classes/data_source_stats.py             22      9    59%   61, 64, 73-79
+pyaurorax/search/requests/__init__.py                             26      0   100%
+pyaurorax/search/requests/_requests.py                            80      0   100%
+pyaurorax/search/sources/__init__.py                              59      0   100%
+pyaurorax/search/sources/_sources.py                             110      0   100%
+pyaurorax/search/sources/classes/data_source.py                   66      0   100%
+pyaurorax/search/sources/classes/data_source_stats.py             22      0   100%
 pyaurorax/search/util/__init__.py                                 11      0   100%
 pyaurorax/search/util/_calculate_btrace.py                        18      0   100%
 pyaurorax/tools/__init__.py                                       61     12    80%   93, 100, 107, 114, 121, 128, 135, 142, 222, 256, 298, 313
@@ -94,7 +94,7 @@ pyaurorax/tools/mosaic/_prep_skymaps.py                          119    112
 pyaurorax/tools/spectra/__init__.py                               10      1    90%   129
 pyaurorax/tools/spectra/_plot.py                                 114    108     5%   42-233
 --------------------------------------------------------------------------------------------
-TOTAL                                                           5863   2247    62%
+TOTAL                                                           5833   2147    63%
 
 13 empty files skipped.
 ```

pyaurorax/pyaurorax.py

@@ -18,7 +18,7 @@
 import pyucalgarysrs
 from texttable import Texttable
 from pathlib import Path
-from typing import Optional, Dict, Any, Literal
+from typing import Optional, Any, Literal
 from . import __version__
 from .exceptions import AuroraXInitializationError, AuroraXPurgeError
 from .search import SearchManager
@@ -60,7 +60,6 @@ def __init__(
         read_tar_temp_path: Optional[str] = None,
         api_base_url: Optional[str] = None,
         api_timeout: Optional[int] = None,
-        api_headers: Optional[Dict] = None,
         api_key: Optional[str] = None,
         progress_bar_backend: Literal["auto", "standard", "notebook"] = "auto",
     ):
@@ -86,11 +85,6 @@ def __init__(
                 The timeout used when communicating with the Aurorax API. This value is represented in 
                 seconds, and by default is `10 seconds`.
             
-            api_headers (Dict): 
-                HTTP headers used when communicating with the AuroraX API. The default for this value 
-                consists of several standard headers. Any changes to this parameter are in addition to 
-                the default standard headers.
-
             api_key (str): 
                 API key to use when interacting with the AuroraX API. The default value is None. Please note
                 that an API key is only required for write operations to the AuroraX search API, such as
@@ -234,9 +228,18 @@ def api_key(self):
         return self.__api_key
 
     @api_key.setter
-    def api_key(self, value: str):
+    def api_key(self, value: Optional[str] = None):
+        # set the private var
         self.__api_key = value
 
+        # update the headers
+        if (value is None or value == "") and "x-aurorax-api-key" in self.__api_headers:
+            # remove it
+            del self.__api_headers["x-aurorax-api-key"]
+        else:
+            # add/update it
+            self.__api_headers["x-aurorax-api-key"] = self.__api_key  # type: ignore
+
     @property
     def download_output_root_path(self):
         """
@@ -297,14 +300,14 @@ def __str__(self) -> str:
 
     def __repr__(self) -> str:
         return ("PyAuroraX(download_output_root_path='%s', read_tar_temp_path='%s', api_base_url='%s', " +
-                "api_headers=%s, api_timeout=%s, api_key='%s', progress_bar_backend='%s', srs_obj=PyUCalgarySRS(...))") % (
+                "api_headers=%s, api_timeout=%s, api_key=%s, progress_bar_backend='%s', srs_obj=PyUCalgarySRS(...))") % (
                     self.__download_output_root_path,
                     self.__read_tar_temp_path,
                     self.api_base_url,
                     self.api_headers,
                     self.api_timeout,
+                    "None" if self.api_key is None else "'%s'" % (self.api_key),
                     self.progress_bar_backend,
-                    self.api_key,
                 )
 
     def pretty_print(self):

pyaurorax/search/requests/__init__.py

@@ -21,7 +21,7 @@
 """
 
 import datetime
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Literal
 from ._requests import get_status as func_get_status
 from ._requests import get_data as func_get_data
 from ._requests import get_logs as func_get_logs
@@ -144,7 +144,7 @@ def cancel(self, request_url: str, wait: bool = False, poll_interval: float = __
         return func_cancel(self.__aurorax_obj, request_url, wait, poll_interval, verbose)
 
     def list(self,
-             search_type: Optional[str] = None,
+             search_type: Optional[Literal["conjunction", "ephemeris", "data_product"]] = None,
              active: Optional[bool] = None,
              start: Optional[datetime.datetime] = None,
              end: Optional[datetime.datetime] = None,
@@ -186,11 +186,12 @@ def list(self,
             List of matching search requests
 
         Raises:
+            ValueError: Unsupported search type
             pyaurorax.exceptions.AuroraXUnauthorizedError: Invalid API key for this operation
         """
         return func_list(self.__aurorax_obj, search_type, active, start, end, file_size, result_count, query_duration, error_condition)
 
-    def delete(self, request_id: str) -> int:
+    def delete(self, request_id: str) -> int:  # pragma: nocover
         """
         Entirely remove a search request from the AuroraX database. Administrators only.
 

pyaurorax/search/requests/_requests.py

@@ -17,7 +17,6 @@
 
 import datetime
 import time
-from ..._util import show_warning
 from ..api.classes.request import AuroraXAPIRequest
 from ..location import Location
 from ...exceptions import (
@@ -46,11 +45,11 @@ def get_data(aurorax_obj, data_url, response_format, skip_serializing):
         else:
             req = AuroraXAPIRequest(aurorax_obj, method="get", url=data_url)
         res = req.execute()
-    except Exception as e:
+    except Exception as e:  # pragma: nocover
         raise AuroraXDataRetrievalError("unable to retrieve data (likely there is none)") from e
 
     # check for error message
-    if ("error" in res.data):
+    if ("error" in res.data):  # pragma: nocover
         raise AuroraXDataRetrievalError("%s: %s" % (
             res.data["error"]["error_code"],
             res.data["error"]["error_message"],
@@ -84,7 +83,7 @@ def get_logs(aurorax_obj, request_url):
     # return
     if ("logs" in status):
         return status["logs"]
-    else:
+    else:  # pragma: nocover
         return []
 
 
@@ -118,7 +117,7 @@ def cancel(aurorax_obj, request_url, wait, poll_interval, verbose):
     status = get_status(aurorax_obj, request_url)
 
     # wait for request to be cancelled
-    while (status["search_result"]["data_uri"] is None and status["search_result"]["error_condition"] is False):
+    while (status["search_result"]["data_uri"] is None and status["search_result"]["error_condition"] is False):  # pragma: nocover
         time.sleep(poll_interval)
         if (verbose is True):
             print("[%s] Checking for cancellation status ..." % (datetime.datetime.now()))
@@ -133,10 +132,8 @@ def cancel(aurorax_obj, request_url, wait, poll_interval, verbose):
 def list(aurorax_obj, search_type, active, start, end, file_size, result_count, query_duration, error_condition):
     # check the search request type
     if (search_type is not None and search_type not in __ALLOWED_SEARCH_LISTING_TYPES):
-        show_warning("The search type value '%s' is not one that PyAuroraX knows about. Supported values are: "
-                     "%s. Aborting request." % (search_type, ', '.join(__ALLOWED_SEARCH_LISTING_TYPES)),
-                     stacklevel=1)
-        return []
+        raise ValueError("The search type value '%s' is not one that PyAuroraX knows about. Supported values are: "
+                         "%s. Aborting request." % (search_type, ', '.join(__ALLOWED_SEARCH_LISTING_TYPES)))
 
     # set params
     params = {}
@@ -158,21 +155,18 @@ def list(aurorax_obj, search_type, active, start, end, file_size, result_count,
         params["query_duration"] = query_duration
 
     # do request
-    url = "%s/%s" % (aurorax_obj.api_base_url, aurorax_obj.search.api.URL_SUFFIX_LIST_REQUESTS)
-    req = AuroraXAPIRequest(aurorax_obj, method="get", url=url, params=params)
-    res = req.execute()
-
-    # check responses
-    if (res.status_code == 401):
-        raise AuroraXUnauthorizedError("API key not detected, please authenticate first.")
-    if (res.status_code == 403):
-        raise AuroraXUnauthorizedError("Administrator account required. API key not valid for this level of access")
+    try:
+        url = "%s/%s" % (aurorax_obj.api_base_url, aurorax_obj.search.api.URL_SUFFIX_LIST_REQUESTS)
+        req = AuroraXAPIRequest(aurorax_obj, method="get", url=url, params=params)
+        res = req.execute()
+    except AuroraXUnauthorizedError as e:
+        raise AuroraXUnauthorizedError("An Administrator API key was not detected, and is required for this function") from e
 
     # return
     return res.data
 
 
-def delete(aurorax_obj, request_id):
+def delete(aurorax_obj, request_id):  # pragma: nocover
     # do request
     url = "%s/%s" % (aurorax_obj.api_base_url, aurorax_obj.search.api.URL_SUFFIX_DELETE_REQUESTS.format(request_id))
     req = AuroraXAPIRequest(aurorax_obj, method="delete", url=url, null_response=True)

pyaurorax/search/sources/__init__.py

@@ -424,12 +424,14 @@ def add(self, data_source: DataSource) -> DataSource:
 
         Args:
             data_source (DataSource): 
-                The data source to add (note: it must be a fully-defined DataSource object)
+                The data source to add. THe data source record must have at least the following 
+                values specified: program, platform, instrument_type, display_name, and source_type.
 
         Returns:
             The newly created `DataSource`.
 
         Raises:
+            pyaurorax.exceptions.ValueError: Invalid values for DataSource supplied
             pyaurorax.exceptions.AuroraXAPIError: Error during API call
             pyaurorax.exceptions.AuroraXUnauthorizedError: Not allowed to perform task, or API key / user permissions are invalid
             pyaurorax.exceptions.AuroraXDuplicateError: Duplicate data source, already exists

pyaurorax/search/sources/_sources.py

@@ -150,7 +150,7 @@ def get(aurorax_obj, program, platform, instrument_type, format, include_stats):
     # set results to the first thing found
     if (len(sources) == 1):
         return sources[0]
-    elif (len(sources) > 1):
+    elif (len(sources) > 1):  # pragma: nocover-ok
         show_warning("Found more than one data source matching this criteria, returning the first (found %d)" % (len(sources)), stacklevel=1)
         return sources[0]
     else:
@@ -205,16 +205,16 @@ def add(aurorax_obj, ds):
     # note that the source type also has a limit, but because we control these
     # ourselves, we don't need a check for it.
     if not all([ds.program, ds.platform, ds.instrument_type, ds.source_type, ds.display_name]):
-        raise AuroraXAPIError("Missing required fields. To create a data source, the program, platform, " +
-                              "instrument_type, source_type, and display_name, are all required")
+        raise ValueError("Missing required fields. To create a data source, the program, platform, " +
+                         "instrument_type, source_type, and display_name, are all required")
     if (len(ds.program) > 200):
-        raise AuroraXAPIError("Program too long. Must be less than 200 characters")
+        raise ValueError("Program too long. Must be less than 200 characters")
     if (len(ds.platform) > 50):
-        raise AuroraXAPIError("Platform too long. Must be less than 50 characters")
+        raise ValueError("Platform too long. Must be less than 50 characters")
     if (len(ds.instrument_type) > 200):
-        raise AuroraXAPIError("Instrument type too long. Must be less than 200 characters")
+        raise ValueError("Instrument type too long. Must be less than 200 characters")
     if (len(ds.display_name) > 50):
-        raise AuroraXAPIError("Display name too long. Must be less than 50 characters")
+        raise ValueError("Display name too long. Must be less than 50 characters")
 
     # set up request
     request_data = {
@@ -242,7 +242,7 @@ def add(aurorax_obj, ds):
     # return
     try:
         return DataSource(**res.data)
-    except Exception:
+    except Exception:  # pragma: nocover
         raise AuroraXAPIError("Could not create data source") from None
 
 
@@ -253,7 +253,7 @@ def delete(aurorax_obj, identifier):
     res = req.execute()
 
     # evaluate response
-    if (res.status_code == 400):
+    if (res.status_code == 400):  # pragma: nocover
         raise AuroraXAPIError("%s - %s" % (res.data["error_code"], res.data["error_message"]))
     elif (res.status_code == 409):
         raise AuroraXConflictError("%s - %s" % (res.data["error_code"], res.data["error_message"]))
@@ -290,5 +290,5 @@ def update(aurorax_obj, identifier, program, platform, instrument_type, source_t
     # return
     try:
         return get_using_identifier(aurorax_obj, ds.identifier, FORMAT_FULL_RECORD, False)
-    except Exception:
+    except Exception:  # pragma: nocover
         raise AuroraXAPIError("Could not update data source") from None

pyaurorax/search/sources/classes/data_source.py

@@ -204,7 +204,7 @@ def pretty_print(self):
         else:
             data_product_metadata_schema_str = self.data_product_metadata_schema
         print("  %-30s: %s" % ("data_product_metadata_schema", data_product_metadata_schema_str))
-        if (self.stats is not None and len(str(self.stats)) > max_len):
+        if (self.stats is not None and len(str(self.stats)) > max_len):  # pragma: nocover
             stats_str = "%s..." % (str(self.stats)[0:max_len])
         else:
             stats_str = self.data_product_metadata_schema

tests/test_suite/conftest.py

@@ -33,6 +33,7 @@
 def pytest_addoption(parser):
     parser.addoption("--api-url", action="store", default="https://api.staging.aurorax.space", help="A specific API URL to use")
     parser.addoption("--api-key", type=str, help="A specific API key to use")
+    parser.addoption("--no-init-tasks", action="store_true", help="Do not do the initialization tasks")
 
 
 #---------------------------------------------------
@@ -301,22 +302,25 @@ def setup_task4():
         setup_task_dict["data_products_search_id"] = s.request_id
 
     # create and run threads for each setup task
-    print("[SETUP] Running setup tasks ...")
-    thread1 = threading.Thread(target=setup_task1)
-    thread2 = threading.Thread(target=setup_task2)
-    thread3 = threading.Thread(target=setup_task3)
-    thread4 = threading.Thread(target=setup_task4)
-    thread1.start()
-    thread2.start()
-    thread3.start()
-    thread4.start()
-    thread1.join()
-    thread2.join()
-    thread3.join()
-    thread4.join()
-    CONJUNCTION_SEARCH_REQUEST_ID = setup_task_dict["conjunction_search_id"]
-    EPHEMERIS_SEARCH_REQUEST_ID = setup_task_dict["ephemeris_search_id"]
-    DATA_PRODUCTS_SEARCH_REQUEST_ID = setup_task_dict["data_products_search_id"]
+    if (session.config.getoption("--no-init-tasks") is False):
+        print("[SETUP] Running setup tasks ...")
+        thread1 = threading.Thread(target=setup_task1)
+        thread2 = threading.Thread(target=setup_task2)
+        thread3 = threading.Thread(target=setup_task3)
+        thread4 = threading.Thread(target=setup_task4)
+        thread1.start()
+        thread2.start()
+        thread3.start()
+        thread4.start()
+        thread1.join()
+        thread2.join()
+        thread3.join()
+        thread4.join()
+        CONJUNCTION_SEARCH_REQUEST_ID = setup_task_dict["conjunction_search_id"]
+        EPHEMERIS_SEARCH_REQUEST_ID = setup_task_dict["ephemeris_search_id"]
+        DATA_PRODUCTS_SEARCH_REQUEST_ID = setup_task_dict["data_products_search_id"]
+    else:
+        print("[SETUP] Skipping setup tasks")
 
     # complete
     print("[SETUP] Initialization completed in %s" % (datetime.datetime.now() - d1))
@@ -327,7 +331,7 @@ def pytest_sessionfinish(session, exitstatus):
     Called after whole test run finished, right before
     returning the exit status to the system.
     """
-    print("\n[TEARDOWN] Cleaning up all testing data dirs ...")
+    print("\n\n[TEARDOWN] Cleaning up all testing data dirs ...")
     # delete all data testing dirs
     glob_str = "%s/pyaurorax_data_*testing*" % (str(Path.home()))
     path_list = sorted(glob.glob(glob_str))

tests/test_suite/search/ephemeris/test_search.py

@@ -223,7 +223,7 @@ def test_metadata_filter_warning(aurorax):
 @pytest.mark.search_ro
 def test_cancel(aurorax):
     start_dt = datetime.datetime(2018, 1, 1, 0, 0, 0)
-    end_dt = datetime.datetime(2021, 12, 31, 23, 59, 59)
+    end_dt = datetime.datetime(2018, 1, 31, 23, 59, 59)
     programs = ["themis"]
 
     # do search
@@ -235,3 +235,17 @@ def test_cancel(aurorax):
 
     # check it was cancelled
     assert result == 0
+
+
+@pytest.mark.search_ro
+def test_cancel_no_wait(aurorax):
+    start_dt = datetime.datetime(2018, 1, 1, 0, 0, 0)
+    end_dt = datetime.datetime(2018, 1, 31, 23, 59, 59)
+    programs = ["themis"]
+
+    # do search
+    s = EphemerisSearch(aurorax, start=start_dt, end=end_dt, programs=programs)
+    s.execute()
+
+    # cancel it
+    s.cancel(wait=False)

tests/test_suite/search/requests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 University of Calgary
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.