From 2031277844319acea0dad7f5e0739007e9e030a8 Mon Sep 17 00:00:00 2001 From: Alexander King Date: Tue, 10 Jan 2023 13:36:00 -0600 Subject: [PATCH 1/6] Remove summary_by_domain default rate limit --- src/admiral/certs/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/admiral/certs/tasks.py b/src/admiral/certs/tasks.py index f15db57..7306031 100644 --- a/src/admiral/certs/tasks.py +++ b/src/admiral/certs/tasks.py @@ -28,7 +28,6 @@ @shared_task( autoretry_for=(Exception, requests.HTTPError, requests.exceptions.HTTPError), - rate_limit="10/h", retry_backoff=True, retry_jitter=True, retry_kwargs={"max_retries": 16}, From 826a64f6cc68c42c5834d03a40d5a3e7418da1fb Mon Sep 17 00:00:00 2001 From: Alexander King Date: Tue, 10 Jan 2023 13:41:28 -0600 Subject: [PATCH 2/6] Alphabetize celery-defaults section --- secrets/admiral.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/secrets/admiral.yml b/secrets/admiral.yml index 9e0a7cb..298831b 100644 --- a/secrets/admiral.yml +++ b/secrets/admiral.yml @@ -8,18 +8,8 @@ celery-defaults: &celery-defaults result_backend: redis://:fruitcake@redis:6379/0 result_expires: 3600 task_acks_late: true - task_reject_on_worker_lost: true - task_track_started: true - task_send_sent_event: true - task_default_queue: cyhy_default task_default_exchange: null - task_routes: - admiral.certs.*: - queue: cyhy_cert_work - admiral.port_scan.*: - queue: cyhy_scanner_work - admiral.tester.*: - queue: cyhy_test_work + task_default_queue: cyhy_default task_queues: cyhy_cert_work: routing_key: cyhy_cert_work @@ -27,6 +17,16 @@ celery-defaults: &celery-defaults routing_key: cyhy_scanner_work cyhy_test_work: routing_key: cyhy_test_work + task_reject_on_worker_lost: true + task_routes: + admiral.certs.*: + queue: cyhy_cert_work + admiral.port_scan.*: + queue: cyhy_scanner_work + admiral.tester.*: + queue: cyhy_test_work + task_send_sent_event: true + task_track_started: true # used in the development container dev-mode: From d59c340d0fac65a6c2bd6b0a114d7c3b2eb2e5c3 Mon Sep 17 00:00:00 2001 From: Alexander King Date: Thu, 26 Jan 2023 10:39:37 -0600 Subject: [PATCH 3/6] Make task rate limit configurable With this change, we can limit the number of requests performed within a given time frame. We can specify seconds, minutes, or hours over which tasks will be evenly distributed. --- secrets/admiral.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/secrets/admiral.yml b/secrets/admiral.yml index 298831b..67aa8c4 100644 --- a/secrets/admiral.yml +++ b/secrets/admiral.yml @@ -37,9 +37,15 @@ dev-mode: - admiral.port_scan - admiral.tester +task_rate_limit: &task_rate_limit + # Modify the rate below as needed. "10/h" means limit to 10 request per hour. + rate_limit: '10/h' + cert-worker: &default-section celery: <<: *celery-defaults + task_annotations: + tasks.summary_by_domain: *task_rate_limit task_default_queue: cyhy_cert_work task_queues: cyhy_cert_work: From 02cf0ce988ec0a9a34057a6acf3549185d6aceb7 Mon Sep 17 00:00:00 2001 From: Alexander King Date: Mon, 30 Jan 2023 12:54:36 -0600 Subject: [PATCH 4/6] Establish new default rate limit This commit establishes a new default rate limit for the summary_by_domain task. Since the rate limit is applied per worker, I reduced the limit to 2 tasks per hour and the number of cert-worker replicas to 5 (for easy math). This way, we can match the rate limit for the Small Tier of the Certificate Transparency Search API. --- docker-compose.yml | 2 +- secrets/admiral.yml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e170ebe..3d9670d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ x-admiral-template: &admiral-template dockerfile: Dockerfile-admiral deploy: mode: replicated - replicas: 6 + replicas: 5 environment: ADMIRAL_CONFIG_FILE: "/run/secrets/admiral.yml" ADMIRAL_CONFIG_SECTION: dev-mode diff --git a/secrets/admiral.yml b/secrets/admiral.yml index 67aa8c4..a4ab873 100644 --- a/secrets/admiral.yml +++ b/secrets/admiral.yml @@ -38,8 +38,10 @@ dev-mode: - admiral.tester task_rate_limit: &task_rate_limit - # Modify the rate below as needed. "10/h" means limit to 10 request per hour. - rate_limit: '10/h' + # Modify the rate below as needed. Limits are applied per worker, so + # "2/h" means limit one worker to 2 tasks per hour. To get the global + # rate limit, you must multiply by the number of workers provisioned. + rate_limit: "2/h" cert-worker: &default-section celery: From 58755315a87eea9c50ed11c21532e397bea0f762 Mon Sep 17 00:00:00 2001 From: Alexander King Date: Tue, 31 Jan 2023 11:39:31 -0600 Subject: [PATCH 5/6] Include rate limit syntax reference --- secrets/admiral.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/secrets/admiral.yml b/secrets/admiral.yml index a4ab873..f9fe2e6 100644 --- a/secrets/admiral.yml +++ b/secrets/admiral.yml @@ -41,6 +41,8 @@ task_rate_limit: &task_rate_limit # Modify the rate below as needed. Limits are applied per worker, so # "2/h" means limit one worker to 2 tasks per hour. To get the global # rate limit, you must multiply by the number of workers provisioned. + # For more syntax examples, refer to + # https://docs.celeryq.dev/en/stable/userguide/tasks.html#Task.rate_limit rate_limit: "2/h" cert-worker: &default-section From 78b1cfae5a96da0a2db6c11926bbef2cf48bddbf Mon Sep 17 00:00:00 2001 From: Alexander King Date: Wed, 1 Feb 2023 15:38:37 -0600 Subject: [PATCH 6/6] Bump version from 1.2.0 to 1.3.0 --- src/admiral/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admiral/_version.py b/src/admiral/_version.py index 6482454..b58ac8d 100644 --- a/src/admiral/_version.py +++ b/src/admiral/_version.py @@ -1,2 +1,2 @@ """This file defines the version of this module.""" -__version__ = "1.2.0" +__version__ = "1.3.0"