Make task rate limiting opt in

configs/test/project.yaml

@@ -33,6 +33,9 @@ coverage:
     # Bucket to load code coverage information from.
     bucket: test-coverage-bucket
 
+rate_limit_tasks:
+  enabled: true
+
 logs:
   fuzzer:
     # Bucket to store logs for fuzzer runs.

src/clusterfuzz/_internal/base/tasks/task_rate_limiting.py

@@ -15,6 +15,8 @@
 
 import datetime
 
+from clusterfuzz._internal.base import memoize
+from clusterfuzz._internal.config import local_config
 from clusterfuzz._internal.datastore import data_types
 from clusterfuzz._internal.datastore import ndb_utils
 from clusterfuzz._internal.metrics import logs
@@ -33,6 +35,13 @@ def _get_datetime_now():
 _UTASK_PSEUDO_TASKS = {'uworker_main', 'postprocess', 'preprocess'}
 
 
+@memoize.wrap(memoize.FifoInMemory(1))
+def optin_to_task_rate_limiting():
+  enabled = local_config.ProjectConfig().get('rate_limit_tasks.enabled', False)
+  logs.info(f'Using async HTTP: {enabled}.')
+  return enabled
+
+
 class TaskRateLimiter:
   """Rate limiter for tasks. This limits tasks to 100 erroneous runs or 2000
   succesful runs in 6 hours. It keeps track of task completion when record_task
@@ -51,6 +60,8 @@ def __str__(self):
 
   def record_task(self, success: bool) -> None:
     """Records a task and whether it completed succesfully."""
+    if not optin_to_task_rate_limiting():
+      return
     if self.task_name in _UTASK_PSEUDO_TASKS:
       # Don't rate limit these fake uworker tasks.
       return
@@ -67,6 +78,8 @@ def record_task(self, success: bool) -> None:
 
   def is_rate_limited(self) -> bool:
     """Checks if the given task is rate limited."""
+    if not optin_to_task_rate_limiting():
+      return False
     if self.task_name in _UTASK_PSEUDO_TASKS:
       # Don't rate limit these fake tasks.
       return False