Add possibility to drop at an interval even if QueryOnMissing=preserve

Dockerfile

@@ -1,16 +1,25 @@
-FROM python:3.8-slim
+FROM remote-docker.artifactory.swisscom.com/python:3.8-slim
 
-WORKDIR /usr/src/app
+RUN pip install --upgrade pip
 
-COPY setup.py /usr/src/app/
-COPY README.md /usr/src/app/
+RUN groupadd --gid 1000 worker && useradd --uid 1000 --gid 1000 -m worker
+
+USER 1000
+WORKDIR /home/worker
+
+RUN pip install --user pipenv
+ENV PATH="/home/worker/.local/bin:${PATH}"
+
+COPY --chown=worker:worker setup.py /home/worker
+COPY --chown=worker:worker README.md /home/worker
 # Elasticsearch switched to a non open source license from version 7.11 onwards.
 # Limit to earlier versions to avoid license and compatibility issues.
-RUN pip install -e . 'elasticsearch<7.11'
+RUN pip install --user -e . 'elasticsearch<7.11'
+RUN pip install --user -e . 'wsgi_basic_auth'
 
-COPY prometheus_es_exporter/*.py /usr/src/app/prometheus_es_exporter/
-COPY LICENSE /usr/src/app/
+COPY --chown=worker:worker prometheus_es_exporter/*.py /home/worker/prometheus_es_exporter/
+COPY --chown=worker:worker LICENSE /home/worker
 
 EXPOSE 9206
 
-ENTRYPOINT ["python", "-u", "/usr/local/bin/prometheus-es-exporter"]
+ENTRYPOINT ["python", "-u", "/home/worker/prometheus_es_exporter"]

README.md

@@ -70,6 +70,8 @@ By default, it will bind to port 9206, query Elasticsearch on `localhost:9200` a
 ```
 Run with the `-h` flag to see details on all the available options.
 
+Set the environment var `WSGI_AUTH_CREDENTIALS` to `user:password` to protect the metrics endpoint with a basic auth user. E.g. 'foo:bar'. Multiple credentials are separated with a | (pipe) character.
+
 Note that all options can be set via environment variables. The environment variable names are prefixed with `ES_EXPORTER`, e.g. `ES_EXPORTER_BASIC_USER=fred` is equivalent to `--basic-user fred`. CLI options take precedence over environment variables.
 
 Command line options can also be set from a configuration file, by passing `--config FILE`. The format of the file should be [Configobj's unrepre mode](https://configobj.readthedocs.io/en/latest/configobj.html#unrepr-mode), so instead of `--basic-user fred` you could use a configuration file `config_file` with `basic-user="fred"` in it, and pass `--config config_file`. CLI options and environment variables take precedence over configuration files.

exporter.cfg

@@ -21,6 +21,9 @@ QueryOnError = drop
 # * drop - remove the metric.
 # * zero - keep the metric, but reset its value to 0.
 QueryOnMissing = drop
+# How often to drop the values even if missing was set to preserve?
+# Applied only when QueryOnMissing = preserve
+QueryOnMissingPreserveButDropAfterSecs = 0
 
 # Queries are defined in sections beginning with 'query_'.
 # Characters following this prefix will be used as a prefix for all metrics

prometheus_es_exporter/__init__.py

@@ -12,7 +12,10 @@
 from elasticsearch import Elasticsearch
 from elasticsearch.exceptions import ConnectionTimeout
 from jog import JogFormatter
-from prometheus_client import start_http_server
+from prometheus_client import make_wsgi_app
+from wsgiref.simple_server import make_server
+from wsgi_basic_auth import BasicAuth
+from threading import Thread
 from prometheus_client.core import GaugeMetricFamily, REGISTRY
 
 from . import cluster_health_parser
@@ -199,6 +202,10 @@ def collect(self):
         for metric_dict in query_metrics.values():
             yield from gauge_generator(metric_dict)
 
+def drop_after(query_name):
+
+    log.info("Clearing the cache...")
+    METRICS_BY_QUERY[query_name] = {}
 
 def run_query(es_client, query_name, indices, query,
               timeout, on_error, on_missing):
@@ -602,18 +609,26 @@ def cli(**options):
                                           fallback='drop')
                 on_missing = config.getenum(section, 'QueryOnMissing',
                                             fallback='drop')
+                drop_after_missing_preserve = config.getfloat(section, 'QueryOnMissingPreserveButDropAfterSecs',
+                                            fallback=0)
 
                 queries[query_name] = (interval, timeout, indices, query,
-                                       on_error, on_missing)
+                                       on_error, on_missing, drop_after_missing_preserve)
 
         scheduler = sched.scheduler()
 
         if queries:
             for query_name, (interval, timeout, indices, query,
-                             on_error, on_missing) in queries.items():
+                             on_error, on_missing, drop_after_missing_preserve) in queries.items():
+                """ If drop after is set, setup a scheduler to clear the cache """
+                if on_missing == 'preserve' and drop_after_missing_preserve > 0:
+                    schedule_job(scheduler, executor, drop_after_missing_preserve,
+                                                 drop_after, query_name)
+
                 schedule_job(scheduler, executor, interval,
                              run_query, es_client, query_name, indices, query,
                              timeout, on_error, on_missing)
+
         else:
             log.error('No queries found in config file(s)')
             return
@@ -649,7 +664,12 @@ def cli(**options):
         REGISTRY.register(QueryMetricCollector())
 
     log.info('Starting server...')
-    start_http_server(port)
+    app = make_wsgi_app()
+    app = BasicAuth(app)
+    httpd = make_server('', port, app)
+    t = Thread(target=httpd.serve_forever)
+    t.daemon = True
+    t.start()
     log.info('Server started on port %(port)s', {'port': port})
 
     if scheduler: