Improve and unify ingestion server logging, simplify docker CMDs (#2639)

* Correct and unify the logging for the ingestion server

* Simplify ingestion server command, remove unused docker-compose file

* Remove unused param

* Remove comment

* Remove logging config from indexer worker

* Unify ingestion server and indexer worker config

* Add reference link

docker-compose.yml

@@ -240,7 +240,6 @@ services:
       args: # Automatically inferred from env vars, unless specified
         - INGESTION_PY_VERSION
     image: openverse-ingestion_server
-    command: gunicorn -c ./gunicorn.conf.py
     ports:
       - "50281:8001"
     depends_on:
@@ -266,7 +265,7 @@ services:
       args: # Automatically inferred from env vars, unless specified
         - INGESTION_PY_VERSION
     image: openverse-ingestion_server
-    command: gunicorn -c ./gunicorn_worker.conf.py
+    command: gunicorn indexer_worker:api --bind 0.0.0.0:8002
     expose:
       - "8002"
     depends_on:

ingestion_server/Dockerfile

@@ -78,4 +78,4 @@ COPY --chown=ingestionu . /ingestion_server/
 # - 8002: Gunicorn server for `indexer_worker` Falcon app
 EXPOSE 8001 8002
 
-# CMD is set from Docker Compose
+CMD ["gunicorn", "--bind", "0.0.0.0:8001", "api:api"]

ingestion_server/gunicorn.conf.py

@@ -1,7 +1,44 @@
-bind = ["0.0.0.0:8001"]
 capture_output = True
+accesslog = "-"
+errorlog = "-"
 chdir = "./ingestion_server/"
 timeout = 120
 reload = True
+logconfig_dict = {
+    # NOTE: Most of this is inherited from the default configuration
+    # https://github.com/benoitc/gunicorn/blob/cc2e3835784542e65886cd27f64d444309fbaad0/gunicorn/glogging.py#L48-L86
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "generic": {
+            "format": "[%(asctime)s - %(name)s - %(lineno)3d][%(levelname)s] %(message)s",  # noqa: E501
+        },
+    },
+    "root": {"level": "INFO", "handlers": ["console"]},
+    "loggers": {
+        "gunicorn.error": {
+            "level": "DEBUG",
+            "handlers": ["console"],
+            "propagate": False,  # Prevents default handler from also logging this
+            "qualname": "gunicorn.error",
+        },
+        "gunicorn.access": {
+            "level": "INFO",
+            "handlers": ["console"],
+            "propagate": False,  # Prevents default handler from also logging this
+            "qualname": "gunicorn.access",
+        },
+        "": {
+            "level": "INFO",
+            "handlers": ["console"],
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "generic",
+            "stream": "ext://sys.stdout",
+        },
+    },
+}
 loglevel = "debug"
-wsgi_app = "api:api"

ingestion_server/ingestion_server/api.py

@@ -2,7 +2,6 @@
 
 import logging
 import os
-import sys
 import time
 import uuid
 from multiprocessing import Process, Value
@@ -323,20 +322,9 @@ def on_delete(_, __):
         clear_state()
 
 
-def create_api(log=True):
+def create_api():
     """Create an instance of the Falcon API server."""
 
-    if log:
-        root = logging.getLogger()
-        root.setLevel(logging.DEBUG)
-        handler = logging.StreamHandler(sys.stdout)
-        handler.setLevel(logging.INFO)
-        formatter = logging.Formatter(
-            "%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s"
-        )
-        handler.setFormatter(formatter)
-        root.addHandler(handler)
-
     _api = falcon.App()
 
     task_tracker = TaskTracker()

ingestion_server/ingestion_server/indexer_worker.py

@@ -6,7 +6,6 @@
 """
 
 import logging as log
-import sys
 from multiprocessing import Process
 
 import boto3
@@ -111,15 +110,6 @@ def _self_destruct():
     ec2_client.stop_instances(InstanceIds=[instance_id])
 
 
-root = log.getLogger()
-root.setLevel(log.DEBUG)
-handler = log.StreamHandler(sys.stdout)
-handler.setLevel(log.INFO)
-formatter = log.Formatter(
-    "%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s"
-)
-handler.setFormatter(formatter)
-root.addHandler(handler)
 api = falcon.App()
 api.add_route("/indexing_task", IndexingJobResource())
 api.add_route("/healthcheck", HealthcheckResource())