Dependency stuff (#17588)

* do not '--upgrade' in for deps checking

when un-merged releases add transitive dependencies, this check begins failing until that release is merged and the branch rebased over it.

by skipping --upgrade, we align this check to simply ensuring that the compiled .txt files match what we expect for our specified dependencies.

* add/document helpers for manual dependency upgrades

* Update docs/dev/development/patterns.rst

Co-authored-by: Mike Fiedler <[email protected]>

---------

Co-authored-by: Mike Fiedler <[email protected]>

Author: ewdurbin

Makefile

@@ -114,6 +114,12 @@ licenses: .state/docker-build-base
 deps: .state/docker-build-base
 	docker compose run --rm base bin/deps
 
+deps_upgrade_all: .state/docker-build-base
+	docker compose run --rm base bin/deps-upgrade -a
+
+deps_upgrade_project: .state/docker-build-base
+	docker compose run --rm base bin/deps-upgrade -p $(P)
+
 translations: .state/docker-build-base
 	docker compose run --rm base bin/translations
 
@@ -181,4 +187,4 @@ purge: stop clean
 stop:
 	docker compose stop
 
-.PHONY: default build serve resetdb initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations checkdb
+.PHONY: default build serve resetdb initdb shell dbshell tests dev-docs user-docs deps deps_upgrade_all deps_upgrade_project clean purge debug stop compile-pot runmigrations checkdb

bin/deps

@@ -3,13 +3,13 @@ set -ex
 
 export TMPDIR=$(mktemp -d)
 mkdir -p $TMPDIR/docs
-pip-compile --upgrade -o $TMPDIR/deploy.txt requirements/deploy.in > /dev/null
-pip-compile --upgrade -o $TMPDIR/main.txt requirements/main.in > /dev/null
-pip-compile --upgrade -o $TMPDIR/lint.txt requirements/lint.in > /dev/null
-pip-compile --upgrade -o $TMPDIR/tests.txt requirements/tests.in > /dev/null
-pip-compile --upgrade -o $TMPDIR/docs-dev.txt requirements/docs-dev.in > /dev/null
-pip-compile --upgrade -o $TMPDIR/docs-user.txt requirements/docs-user.in > /dev/null
-pip-compile --upgrade -o $TMPDIR/docs-blog.txt requirements/docs-blog.in > /dev/null
+pip-compile -o $TMPDIR/deploy.txt requirements/deploy.in > /dev/null
+pip-compile -o $TMPDIR/main.txt requirements/main.in > /dev/null
+pip-compile -o $TMPDIR/lint.txt requirements/lint.in > /dev/null
+pip-compile -o $TMPDIR/tests.txt requirements/tests.in > /dev/null
+pip-compile -o $TMPDIR/docs-dev.txt requirements/docs-dev.in > /dev/null
+pip-compile -o $TMPDIR/docs-user.txt requirements/docs-user.in > /dev/null
+pip-compile -o $TMPDIR/docs-blog.txt requirements/docs-blog.in > /dev/null
 python bin/depchecker.py $TMPDIR/deploy.txt requirements/deploy.txt
 python bin/depchecker.py $TMPDIR/main.txt requirements/main.txt
 python bin/depchecker.py $TMPDIR/lint.txt requirements/lint.txt

bin/deps-upgrade

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 [OPTIONS]"
+  echo "Options:"
+  echo "  -h, --help     Display this help message"
+  echo "  -a, --all      Upgrade all dependencies"
+  echo "  -p, --package  PACKAGE_NAME Upgrade a specific package"
+}
+
+has_argument() {
+    [[ ("$1" == *=* && -n ${1#*=}) || ( ! -z "$2" && "$2" != -*)  ]];
+}
+
+extract_argument() {
+  echo "${2:-${1#*=}}"
+}
+
+pip_compile_args=""
+
+handle_options() {
+  while [ $# -gt 0 ]; do
+    case $1 in
+      -h | --help)
+        usage
+        exit 0
+        ;;
+      -a | --all)
+        pip_compile_args="--upgrade"
+        ;;
+      -p | --package*)
+        if ! has_argument $@; then
+          echo "Project name not specified." >&2
+          usage
+          exit 1
+        fi
+
+        pip_compile_args="--upgrade-package $(extract_argument $@)"
+
+        shift
+        ;;
+      *)
+        echo "Invalid option: $1" >&2
+        usage
+        exit 1
+        ;;
+    esac
+    shift
+  done
+}
+
+# Main script execution
+handle_options "$@"
+
+set -ex
+pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/deploy.txt requirements/deploy.in
+pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/main.txt requirements/main.in
+pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/lint.txt requirements/lint.in
+pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/tests.txt requirements/tests.in
+pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/docs-dev.txt requirements/docs-dev.in
+pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/docs-user.txt requirements/docs-user.in
+pip-compile --allow-unsafe --generate-hashes $pip_compile_args --output-file=requirements/docs-blog.txt requirements/docs-blog.in

docs/dev/development/patterns.rst

@@ -22,6 +22,18 @@ Upgrading existing dependencies
 Dependencies are automatically upgraded via `Dependabot pull requests`_, and
 occasionally merged by maintainers.
 
+You can manually upgrade a specific dependency with::
+
+    make deps_upgrade_project P={project_name}
+
+For instance, to upgrade boto3::
+
+    make deps_upgrade_project P=boto3
+
+To upgrade all dependencies, you can use::
+
+    make deps_upgrade_all
+
 Adding new dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~