Changed SDAP startup behavior to wait for all datasets to be ready

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Added `dask` dependency
   - Code cleanup
   - Zarr: Fixed handling of times conversion from xr/np datetimes to Unix timestamps
+- Changed SDAP startup behavior to wait for all datasets to be prepared before accepting HTTP requests
 ### Deprecated
 ### Removed
 - SDAP-493: 

analysis/webservice/webapp.py

@@ -18,6 +18,7 @@
 import logging
 import sys
 import os
+from datetime import datetime
 
 import tornado.web
 from tornado.routing import Rule, RuleRouter, AnyMatches
@@ -27,6 +28,8 @@
 from webservice.nexus_tornado.app_builders import NexusAppBuilder
 from webservice.nexus_tornado.app_builders import RedirectAppBuilder
 
+from nexustiles.nexustiles import NexusTileService
+
 try:
     from importlib.metadata import version as _version
     from importlib.metadata import files as _files
@@ -70,6 +73,8 @@ def inject_args_in_config(args, config):
 
 
 def main():
+    start = datetime.now()
+
     logging.basicConfig(
         level=logging.DEBUG,
         format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
@@ -137,9 +142,16 @@ def main():
     log.info("Starting web server in debug mode: %s" % options.debug)
     server = tornado.web.HTTPServer(router)
     server.listen(options.port)
-    log.info("Starting HTTP listener...")
+    log.info('Waiting for dataset backends to come up...')
+
+    with NexusTileService.DS_LOCK:
+        if not NexusTileService.is_update_thread_alive():
+            log.critical('A fatal error occurred when loading the datasets')
+            exit(-1)
+
+    log.info(f"SDAP started in {datetime.now() - start}. Starting HTTP listener...")
     tornado.ioloop.IOLoop.current().start()
 
 
 if __name__ == "__main__":
-    main()
+    main()

data-access/nexustiles/nexustiles.py

@@ -107,7 +107,6 @@ def wrapper(*args, **kwargs):
 
 
 SOLR_LOCK = threading.Lock()
-DS_LOCK = threading.Lock()
 thread_local = threading.local()
 
 
@@ -126,12 +125,14 @@ class NexusTileService:
 
     ds_config = None
 
+    DS_LOCK = threading.Lock()
+
     __update_thread = None
 
     @staticmethod
     def __update_datasets_loop():
         while True:
-            with DS_LOCK:
+            with NexusTileService.DS_LOCK:
                 NexusTileService._update_datasets()
             sleep(3600)
 
@@ -164,12 +165,16 @@ def __init__(self, config=None):
 
             NexusTileService.__update_thread.start()
 
+    @staticmethod
+    def is_update_thread_alive() -> bool:
+        return NexusTileService.__update_thread is not None and NexusTileService.__update_thread.is_alive()
+
     @staticmethod
     def _get_backend(dataset_s) -> AbstractTileService:
         if dataset_s is not None:
             dataset_s = dataset_s
 
-        with DS_LOCK:
+        with NexusTileService.DS_LOCK:
             if dataset_s not in NexusTileService.backends:
                 logger.warning(f'Dataset {dataset_s} not currently loaded. Checking to see if it was recently'
                                f'added')
@@ -300,7 +305,7 @@ def user_ds_update(name, config):
 
         logger.info(f'Updated dataset {name} in Solr. Updating backends')
 
-        with DS_LOCK:
+        with NexusTileService.DS_LOCK:
             NexusTileService._update_datasets()
 
         return {'success': True}
@@ -332,7 +337,7 @@ def user_ds_add(name, path, config, type='zarr'):
 
         logger.info(f'Added dataset {name} to Solr. Updating backends')
 
-        with DS_LOCK:
+        with NexusTileService.DS_LOCK:
             NexusTileService._update_datasets()
 
         return {'success': True}
@@ -357,7 +362,7 @@ def user_ds_delete(name):
 
         logger.info(f'Removed dataset {name} from Solr. Updating backends')
 
-        with DS_LOCK:
+        with NexusTileService.DS_LOCK:
             NexusTileService._update_datasets()
 
         return {'success': True}