Add optional caching of GitHub API requests

.gitignore

@@ -6,3 +6,4 @@
 __pycache__/
 github.db
 htmlcov
+github-cache.sqlite

DEVELOPERS.md

@@ -71,7 +71,7 @@ e.g `just metrics prs` to run metrics/tasks/prs.py
 
 All tasks are defined in `metrics/tasks` and must have a `main()` function that takes no arguments.
 
-### Fast debug mode
+### Speeding up development
 
 You can set a flag to trigger a fast mode which only retrieves and handful of PRs
 but allows the main code paths to be tested quickly.
@@ -80,6 +80,23 @@ but allows the main code paths to be tested quickly.
 DEBUG_FAST=t just metrics prs
 ```
 
+Alternatively you can turn on caching of GitHub API requests.
+This is particularly useful when iterating on metric definitions
+without changing the data that we retrieve from the API.
+
+```
+DEBUG_CACHE=t just metrics prs
+```
+
+NB that the cache has no expiry time
+(although it will be bypassed on subsequent runs if `DEBUG_CACHE isn't defined).
+You can clear the cache explicitly.
+
+```
+just clean-cache
+```
+
+
 ## Tests
 Run the tests with:
 ```

justfile

@@ -168,3 +168,7 @@ docker-build env="dev": _env
 docker-run env="dev" *args="": _env
     {{ just_executable() }} docker-build {{ env }}
     docker compose run --rm metrics-{{ env }} {{ args }}
+
+# See DEVELOPERS.md
+clean-cache:
+    rm -f github-cache.sqlite

metrics/github/client.py

@@ -1,13 +1,22 @@
 import json
+import os
 import textwrap
 
 import requests
 import requests.utils
+import requests_cache
 import structlog
 
 
 log = structlog.get_logger()
 
+# See DEVELOPERS.md
+if "DEBUG_CACHE" in os.environ:
+    requests_cache.install_cache(
+        "github-cache",
+        # Turn on caching for POST requests because that's what GraphQL uses
+        allowable_methods=("GET", "HEAD", "POST"),
+    )
 
 session = requests.Session()
 

requirements.prod.in

@@ -1,5 +1,6 @@
   greenlet
   requests
+  requests-cache
   slack-bolt
   sqlalchemy[postgresql_psycopgbinary]
   structlog

requirements.prod.txt

@@ -4,6 +4,16 @@
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements.prod.txt requirements.prod.in
 #
+attrs==24.2.0 \
+    --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \
+    --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2
+    # via
+    #   cattrs
+    #   requests-cache
+cattrs==24.1.2 \
+    --hash=sha256:67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0 \
+    --hash=sha256:8028cfe1ff5382df59dd36474a86e02d817b06eaf8af84555441bac915d2ef85
+    # via requests-cache
 certifi==2024.7.4 \
     --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \
     --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90
@@ -183,6 +193,10 @@ idna==3.7 \
     --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \
     --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0
     # via requests
+platformdirs==4.3.6 \
+    --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
+    --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
+    # via requests-cache
 psycopg[binary]==3.1.18 \
     --hash=sha256:31144d3fb4c17d78094d9e579826f047d4af1da6a10427d91dfcfb6ecdf6f12b \
     --hash=sha256:4d5a0a5a8590906daa58ebd5f3cfc34091377354a1acced269dd10faf55da60e
@@ -257,11 +271,21 @@ psycopg-binary==3.1.18 \
 requests==2.32.3 \
     --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
     --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
+    # via
+    #   -r requirements.prod.in
+    #   requests-cache
+requests-cache==1.2.1 \
+    --hash=sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603 \
+    --hash=sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1
     # via -r requirements.prod.in
 sentry-sdk==2.14.0 \
     --hash=sha256:1e0e2eaf6dad918c7d1e0edac868a7bf20017b177f242cefe2a6bcd47955961d \
     --hash=sha256:b8bc3dc51d06590df1291b7519b85c75e2ced4f28d9ea655b6d54033503b5bf4
     # via -r requirements.prod.in
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+    # via url-normalize
 slack-bolt==1.20.1 \
     --hash=sha256:4657e592339797b9b804547a21e6b35dd8e2cd1eab676bfb23960660aae049fd \
     --hash=sha256:8fa26e72b0e55c18c1d34a73558e7fe2150bdc7c947de780b938fdb1d7e854fe
@@ -331,9 +355,14 @@ typing-extensions==4.11.0 \
     # via
     #   psycopg
     #   sqlalchemy
+url-normalize==1.4.3 \
+    --hash=sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2 \
+    --hash=sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed
+    # via requests-cache
 urllib3==2.2.2 \
     --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \
     --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168
     # via
     #   requests
+    #   requests-cache
     #   sentry-sdk