Add CI/CD and PDM to new indexer worker (#4497)

* Use PDM for idx worker and add to CI/CD

* Try to update all snapshots

Why did this not work???

* Force snapshot update using lowered diff tolerance

The text was such a small portion of the screen that it would only fail _sometimes_ on the default maxPixelDiffRatio but only for the smallest screens, where the text takes up comparatively more of the pixels. On larger screens, it seems to never fail. I will create a follow up issue for this.

.github/actions/load-img/action.yml

@@ -7,7 +7,7 @@ inputs:
     required: true
 
   setup_images:
-    default: "upstream_db ingestion_server catalog api api_nginx frontend"
+    default: "upstream_db ingestion_server catalog_indexer_worker catalog api api_nginx frontend"
     description: Space-separated list of image names
 
 runs:

.github/filters.yml

@@ -22,6 +22,13 @@ ingestion_server:
   - docker/upstream_db/**
   - docker-compose.yml
   - load_sample_data.sh
+indexer_worker:
+  - indexer_worker/**
+  - docker/db/**
+  - docker/es/**
+  - docker/upstream_db/**
+  - docker-compose.yml
+  - load_sample_data.sh
 frontend:
   - frontend/**
   - packages/js/**

.github/workflows/ci_cd.yml

@@ -40,6 +40,7 @@ jobs:
       changes: ${{ steps.paths-filter.outputs.changes }}
       catalog: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'catalog') }}
       ingestion_server: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'ingestion_server') }}
+      indexer_worker: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'indexer_worker') }}
       api: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'api') }}
       frontend: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'frontend') }}
       documentation: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'documentation') }}
@@ -201,6 +202,7 @@ jobs:
             SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }}
             CATALOG_PY_VERSION=${{ steps.prepare-build-args.outputs.catalog_py_version }}
             CATALOG_AIRFLOW_VERSION=${{ steps.prepare-build-args.outputs.catalog_airflow_version }}
+            INDEXER_WORKER_PY_VERSION=${{ steps.prepare-build-args.outputs.indexer_worker_py_version }}
             API_PY_VERSION=${{ steps.prepare-build-args.outputs.api_py_version }}
             INGESTION_PY_VERSION=${{ steps.prepare-build-args.outputs.ingestion_py_version }}
             FRONTEND_NODE_VERSION=${{ steps.prepare-build-args.outputs.frontend_node_version }}
@@ -316,6 +318,69 @@ jobs:
           just catalog/generate-docs dag true
           just catalog/generate-docs media-props true
 
+  ##################
+  # Indexer worker #
+  ##################
+
+  test-indexer-worker:
+    name: Run tests for indexer worker
+    if: |
+      needs.get-changes.outputs.indexer_worker == 'true' ||
+      needs.get-changes.outputs.ci_cd == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    needs:
+      - get-changes
+      - build-images
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup CI env
+        uses: ./.github/actions/setup-env
+        with:
+          setup_python: false
+          # Python is not needed to run the tests.
+          setup_nodejs: false
+          # Node.js is not needed to run the tests.
+          install_recipe: ""
+
+      - name: Load Docker images
+        uses: ./.github/actions/load-img
+        with:
+          run_id: ${{ github.run_id }}
+          setup_images: upstream_db
+
+      # Sets build args specifying versions needed to build Docker image.
+      - name: Prepare build args
+        id: prepare-build-args
+        run: |
+          just versions | tee "$GITHUB_OUTPUT"
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+
+      - name: Build indexer worker dev image
+        uses: docker/build-push-action@v5
+        with:
+          context: indexer_worker
+          target: indexer_worker
+          push: false
+          load: true
+          tags: openverse-catalog_indexer_worker
+          cache-from: type=gha,scope=indexer_worker_dev
+          cache-to: type=gha,scope=indexer_worker_dev
+          build-args: |
+            INDEXER_WORKER_PY_VERSION=${{ steps.prepare-build-args.outputs.indexer_worker_py_version }}
+            PDM_INSTALL_ARGS=--dev
+
+      - name: Run tests
+        run: |
+          just indexer_worker/test
+
   ####################
   # Ingestion server #
   ####################

automations/python/workflows/set_matrix_images.py

@@ -61,6 +61,12 @@ def ser_set(x):
     ),
     "catalog": Include(image="catalog", target="cat"),
     "ingestion_server": Include(image="ingestion_server", target="ing"),
+    "catalog_indexer_worker": Include(
+        image="catalog_indexer_worker",
+        target="indexer_worker",
+        context="indexer_worker",
+        build_args="PDM_INSTALL_ARGS=--prod",
+    ),
     "api": Include(
         image="api",
         target="api",
@@ -94,6 +100,9 @@ def ser_set(x):
 if "ingestion_server" in changes:
     build_matrix["image"] |= {"upstream_db", "ingestion_server", "api"}
     publish_matrix["image"].add("ingestion_server")
+if "indexer_worker" in changes:
+    build_matrix["image"] |= {"upstream_db", "catalog_indexer_worker", "api"}
+    publish_matrix["image"].add("catalog_indexer_worker")
 if "api" in changes:
     build_matrix["image"] |= {"upstream_db", "ingestion_server", "api", "api_nginx"}
     publish_matrix["image"] |= {"api", "api_nginx"}

indexer_worker/.dockerignore

@@ -2,4 +2,5 @@
 *
 !indexer_worker
 !gunicorn*
-!Pipfile*
+!pyproject.toml
+!pdm.lock

indexer_worker/Dockerfile

@@ -1,42 +1,40 @@
 # syntax=docker/dockerfile:1
 
 # Automatically build image using Python version specified in the `Pipfile`.
-ARG CATALOG_PY_VERSION
+ARG INDEXER_WORKER_PY_VERSION
 
 ##################
 # Python builder #
 ##################
 
-FROM docker.io/python:${CATALOG_PY_VERSION} as builder
+FROM docker.io/python:${INDEXER_WORKER_PY_VERSION} as builder
 
 # Container optimizations
 ENV PYTHONUNBUFFERED=1
 ENV PIP_NO_CACHE_DIR=1
 ENV PIP_NO_COLOR=1
 
-# Activate the virtualenv
-ENV PATH="/venv/bin:$PATH"
-
 # - Install system packages needed for building Python dependencies
-# - Create a virtualenv inside `/venv`
-# - Install Pipenv to install Python dependencies
+# - Install PDM to install Python dependencies
 RUN apt-get update \
       && apt-get install -y python3-dev \
       && rm -rf /var/lib/apt/lists/* \
-    && python -m venv /venv \
-    && pip install --upgrade pipenv
+    && pip install pdm~=2.14
 
 # Copy the Pipenv files into the container
-COPY Pipfile Pipfile.lock ./
+COPY pyproject.toml pdm.lock ./
+
+# Pass additional arguments when installing Python packages with PDM
+ARG PDM_INSTALL_ARGS='--no-editable'
 
-# Install Python dependencies system-wide (uses the active virtualenv)
-RUN pipenv install --system --deploy --dev
+# Install Python dependencies into a new virtualenv
+RUN pdm install --check --frozen-lockfile $PDM_INSTALL_ARGS
 
 ####################
 # Indexer worker #
 ####################
 
-FROM docker.io/python:${CATALOG_PY_VERSION}-slim as ing
+FROM docker.io/python:${INDEXER_WORKER_PY_VERSION}-slim as indexer_worker
 
 LABEL org.opencontainers.image.source="https://github.com/WordPress/openverse"
 
@@ -46,7 +44,7 @@ ENV PIP_NO_CACHE_DIR=1
 ENV PIP_NO_COLOR=1
 
 # Activate the virtualenv
-ENV PATH="/venv/bin:$PATH"
+ENV PATH="/.venv/bin:$PATH"
 
 ENV PYTHONPATH="$PYTHONPATH:/indexer_worker/"
 # TLDEXTRACT fails to cache in /home/supervisord, set its cache to /tmp instead
@@ -55,7 +53,7 @@ ENV TLDEXTRACT_CACHE="/tmp/python-tldextract"
 WORKDIR /indexer_worker
 
 # Copy virtualenv from the builder image
-COPY --from=builder /venv /venv
+COPY --from=builder /.venv /.venv
 
 # - Install system packages needed for running Python dependencies
 #   - libpq-dev: required by `psycopg2`