Prometheus metrics support for logins and presigned URLs (#1156)

.secrets.baseline

@@ -268,14 +268,14 @@
         "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1561
+        "line_number": 1569
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1583
+        "line_number": 1593
       }
     ],
     "tests/credentials/google/test_credentials.py": [
@@ -422,5 +422,5 @@
       }
     ]
   },
-  "generated_at": "2024-03-16T00:09:27Z"
+  "generated_at": "2024-07-25T17:19:58Z"
 }

clear_prometheus_multiproc

@@ -4,6 +4,8 @@
 set -ex
 
 rm -Rf $1
-mkdir $1
+mkdir -p $1
 chmod 755 $1
-chown 100:101 $1
+if id -u nginx &>/dev/null; then
+    chown $(id -u nginx):$(id -g nginx) $1
+fi

fence/__init__.py

@@ -1,21 +1,17 @@
 from collections import OrderedDict
 import os
-import tempfile
 from urllib.parse import urljoin
-import flask
-from flask_cors import CORS
-from sqlalchemy.orm import scoped_session
-from flask import current_app
-from werkzeug.local import LocalProxy
 
 from authutils.oauth2.client import OAuthClient
-from cdislogging import get_logger
-from gen3authz.client.arborist.client import ArboristClient
-from flask_wtf.csrf import validate_csrf
-from werkzeug.middleware.dispatcher import DispatcherMiddleware
 from azure.storage.blob import BlobServiceClient
 from azure.core.exceptions import ResourceNotFoundError
-from urllib.parse import urlparse
+from cdislogging import get_logger
+import flask
+from flask_cors import CORS
+from flask_wtf.csrf import validate_csrf
+from gen3authz.client.arborist.client import ArboristClient
+from sqlalchemy.orm import scoped_session
+
 
 # Can't read config yet. Just set to debug for now, else no handlers.
 # Later, in app_config(), will actually set level based on config
@@ -31,6 +27,7 @@
 )
 
 from fence.auth import logout, build_redirect_url
+from fence.metrics import metrics
 from fence.blueprints.data.indexd import S3IndexedFileLocation
 from fence.blueprints.login.utils import allowed_login_redirects, domain
 from fence.errors import UserError
@@ -67,11 +64,6 @@
 import fence.blueprints.ga4gh
 
 
-# for some reason the temp dir does not get created properly if we move
-# this statement to `_setup_prometheus()`
-PROMETHEUS_TMP_COUNTER_DIR = tempfile.TemporaryDirectory()
-
-
 app = flask.Flask(__name__)
 CORS(app=app, headers=["content-type", "accept"], expose_headers="*")
 
@@ -102,6 +94,9 @@ def app_init(
     app_sessions(app)
     app_register_blueprints(app)
     server.init_app(app, query_client=query_client)
+    logger.info(
+        f"Prometheus metrics are{'' if config['ENABLE_PROMETHEUS_METRICS'] else ' NOT'} enabled."
+    )
 
 
 def app_sessions(app):
@@ -206,6 +201,15 @@ def public_keys():
             {"keys": [(keypair.kid, keypair.public_key) for keypair in app.keypairs]}
         )
 
+    @app.route("/metrics")
+    def metrics_endpoint():
+        """
+        /!\ There is no authz control on this endpoint!
+        In cloud-automation setups, access to this endpoint is blocked at the revproxy level.
+        """
+        data, content_type = metrics.get_latest_metrics()
+        return flask.Response(data, content_type=content_type)
+
 
 def _check_azure_storage(app):
     """
@@ -365,13 +369,6 @@ def app_config(
     _setup_data_endpoint_and_boto(app)
     _load_keys(app, root_dir)
 
-    app.prometheus_counters = {}
-    if config["ENABLE_PROMETHEUS_METRICS"]:
-        logger.info("Enabling Prometheus metrics...")
-        _setup_prometheus(app)
-    else:
-        logger.info("Prometheus metrics are NOT enabled.")
-
     app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"], logger=logger)
 
     app.debug = config["DEBUG"]
@@ -495,27 +492,6 @@ def _setup_audit_service_client(app):
     )
 
 
-def _setup_prometheus(app):
-    # This environment variable MUST be declared before importing the
-    # prometheus modules (or unit tests fail)
-    # More details on this awkwardness: https://github.com/prometheus/client_python/issues/250
-    os.environ["prometheus_multiproc_dir"] = PROMETHEUS_TMP_COUNTER_DIR.name
-
-    from prometheus_client import (
-        CollectorRegistry,
-        multiprocess,
-        make_wsgi_app,
-    )
-
-    app.prometheus_registry = CollectorRegistry()
-    multiprocess.MultiProcessCollector(app.prometheus_registry)
-
-    # Add prometheus wsgi middleware to route /metrics requests
-    app.wsgi_app = DispatcherMiddleware(
-        app.wsgi_app, {"/metrics": make_wsgi_app(registry=app.prometheus_registry)}
-    )
-
-
 @app.errorhandler(Exception)
 def handle_error(error):
     """

fence/blueprints/data/indexd.py

@@ -49,6 +49,7 @@
 from fence.resources.ga4gh.passports import sync_gen3_users_authz_from_ga4gh_passports
 from fence.resources.audit.utils import enable_audit_logging
 from fence.utils import get_valid_expiration_from_request
+from fence.metrics import metrics
 
 from . import multipart_upload
 from ...models import AssumeRoleCacheAWS, query_for_user, query_for_user_by_id
@@ -77,6 +78,7 @@ def get_signed_url_for_file(
     ga4gh_passports=None,
     db_session=None,
     bucket=None,
+    drs="False",
 ):
     requested_protocol = requested_protocol or flask.request.args.get("protocol", None)
     r_pays_project = flask.request.args.get("userProject", None)
@@ -164,12 +166,33 @@ def get_signed_url_for_file(
         user_sub=flask.g.audit_data.get("sub", ""),
         client_id=_get_client_id(),
         requested_protocol=requested_protocol,
+        action=action,
+        drs=drs,
     )
 
     return {"url": signed_url}
 
 
-def _log_signed_url_data_info(indexed_file, user_sub, client_id, requested_protocol):
+def get_bucket_from_urls(urls, protocol):
+    """
+    Return the bucket name from the first of the provided URLs that starts with the given protocol (usually `gs`, `s3`, `az`...)
+    """
+    bucket = ""
+    for url in urls:
+        if "://" in url:
+            # Extract the protocol and the rest of the URL
+            bucket_protocol, rest_of_url = url.split("://", 1)
+
+            if bucket_protocol == protocol:
+                # Extract bucket name
+                bucket = f"{bucket_protocol}://{rest_of_url.split('/')[0]}"
+                break
+    return bucket
+
+
+def _log_signed_url_data_info(
+    indexed_file, user_sub, client_id, requested_protocol, action, drs="False"
+):
     size_in_kibibytes = (indexed_file.index_document.get("size") or 0) / 1024
     acl = indexed_file.index_document.get("acl")
     authz = indexed_file.index_document.get("authz")
@@ -180,23 +203,25 @@ def _log_signed_url_data_info(indexed_file, user_sub, client_id, requested_proto
         protocol = indexed_file.indexed_file_locations[0].protocol
 
     # figure out which bucket was used based on the protocol
-    bucket = ""
-    for url in indexed_file.index_document.get("urls", []):
-        bucket_name = None
-        if "://" in url:
-            # Extract the protocol and the rest of the URL
-            bucket_protocol, rest_of_url = url.split("://", 1)
-
-            if bucket_protocol == protocol:
-                # Extract bucket name
-                bucket = f"{bucket_protocol}://{rest_of_url.split('/')[0]}"
-                break
+    bucket = get_bucket_from_urls(indexed_file.index_document.get("urls", []), protocol)
 
     logger.info(
-        f"Signed URL Generated. size_in_kibibytes={size_in_kibibytes} "
+        f"Signed URL Generated. action={action} size_in_kibibytes={size_in_kibibytes} "
         f"acl={acl} authz={authz} bucket={bucket} user_sub={user_sub} client_id={client_id}"
     )
 
+    metrics.add_signed_url_event(
+        action,
+        protocol,
+        acl,
+        authz,
+        bucket,
+        user_sub,
+        client_id,
+        drs,
+        size_in_kibibytes,
+    )
+
 
 def _get_client_id():
     client_id = "Unknown Client"
@@ -208,6 +233,7 @@ def _get_client_id():
 
     return client_id
 
+
 def prepare_presigned_url_audit_log(protocol, indexed_file):
     """
     Store in `flask.g.audit_data` the data needed to record an audit log.

fence/blueprints/ga4gh.py

@@ -41,5 +41,7 @@ def get_ga4gh_signed_url(object_id, access_id):
         object_id,
         requested_protocol=access_id,
         ga4gh_passports=ga4gh_passports,
+        drs="True",
     )
+
     return flask.jsonify(result)

fence/blueprints/login/base.py

@@ -7,6 +7,7 @@
 from fence.blueprints.login.redirect import validate_redirect
 from fence.config import config
 from fence.errors import UserError
+from fence.metrics import metrics
 
 logger = get_logger(__name__)
 
@@ -133,6 +134,14 @@ def get(self):
 
     def post_login(self, user=None, token_result=None, **kwargs):
         prepare_login_log(self.idp_name)
+        metrics.add_login_event(
+            user_sub=flask.g.user.id,
+            idp=self.idp_name,
+            fence_idp=flask.session.get("fence_idp"),
+            shib_idp=flask.session.get("shib_idp"),
+            client_id=flask.session.get("client_id"),
+        )
+
         if token_result:
             username = token_result.get(self.username_field)
             if self.is_mfa_enabled:

fence/blueprints/login/google.py

@@ -25,4 +25,5 @@ def get(self):
                 config.get("BASE_URL", "")
                 + "/link/google/callback?code={}".format(flask.request.args.get("code"))
             )
+
         return super(GoogleCallback, self).get()

fence/config-default.yaml

@@ -62,6 +62,7 @@ MOCK_STORAGE: true
 # WARNING: ONLY set to true when fence will be deployed in such a way that it will
 #          ONLY receive traffic from internal clients and can safely use HTTP.
 AUTHLIB_INSECURE_TRANSPORT: true
+
 # enable Prometheus Metrics for observability purposes
 #
 # WARNING: Any counters, gauges, histograms, etc. should be carefully