Task/WG-383: add queue for heavy tasks (#220)

* Rework worker Dockerfile and bump PotreeConverter

* Fix docker compose commands

* Update and improve custom html template

* Activate conda when starting bash on running container

* Add an additional test

* Improve nginx.conf to allow range requests for potree bin files

* Fix adding of nsf_logo.png

* Fix unit test

* Add vim package

* Do not need to install laszip

* Clean up dockerfile

* Update deployed nginx.conf to allow range requests for potree bin files

* Simplify entrypoint script

* Refactor oder of Dockerfile

* Fix nginx.conf so range requests work on Firefox

* Remove unused background image in template

Also, fixes nsf log snippet application.

* Improve comment

* Fix PYTHONPATH

* Fix .bin settings

* Unify how some settings are set

* Lower max body size

This was high as we used to support direct file upload instead of using TAPIS

* Improve error handling to log memory issues

* Add heavy queue for computationally intensive tasks.

devops/docker-compose.local.yml

@@ -59,7 +59,12 @@ services:
     tty: true
     container_name: geoapiworkers
     hostname: geoapiworkers
-    command: "celery -A geoapi.celery_app worker -l info"
+    command: >
+      sh -c '
+      celery -A geoapi.celery_app worker -l info -Q default -n default_worker@geoapi &
+      celery -A geoapi.celery_app worker -l info -Q heavy --concurrency=6 -n heavy_worker@geoapi &
+      wait
+      '
 
   celerybeat:
     image: taccaci/geoapi-workers:local

devops/geoapi-workers/docker-compose.yml

@@ -18,7 +18,12 @@ services:
       driver: syslog
       options:
         tag: geoapi_workers
-    command: "celery -A geoapi.celery_app worker -l info"
+    command:  >
+      sh -c '
+      celery -A geoapi.celery_app worker -l info -Q default -n default_worker@geoapi &
+      celery -A geoapi.celery_app worker -l info -Q heavy --concurrency=6 -n heavy_worker@geoapi &
+      wait
+      '
 
   watchtower:
     image: containrrr/watchtower:1.7.1

geoapi/celery_app.py

@@ -15,6 +15,14 @@
              broker=CELERY_CONNECTION_STRING,
              include=['geoapi.tasks'])
 
+# Define the queues
+app.conf.task_queues = {
+    'default': {'exchange': 'default', 'routing_key': 'default'},
+    'heavy': {'exchange': 'heavy', 'routing_key': 'heavy'}
+}
+
+app.conf.task_default_queue = 'default'
+
 app.conf.beat_schedule = {
     'refresh_projects_watch_content': {
         'task': 'geoapi.tasks.external_data.refresh_projects_watch_content',

geoapi/tasks/external_data.py

@@ -167,7 +167,7 @@ def _handle_point_cloud_conversion_error(pointCloudId, userId, files, error_desc
                                     f"Processing failed for point cloud ({pointCloudId})!")
 
 
-@app.task(rate_limit="1/s")
+@app.task(queue='heavy')
 def import_point_clouds_from_agave(userId: int, files, pointCloudId: int):
     with create_task_session() as session:
         user = session.query(User).get(userId)

geoapi/tasks/streetview.py

@@ -199,7 +199,7 @@ def check_existing_upload(session, user, streetview_service, task_uuid, system_i
 
 
 # TODO: Ensure that just user works and not userid (previously took userid)
-@app.task(rate_limit="5/s")
+@app.task(queue='heavy')
 def from_tapis_to_streetview(user_id: int,
                              streetview_service_id: int,
                              system_id: str,