refactor: remove upsert_* methods and stop treating data_platform_cat…

…alogue as a separate package (#1229)

* refactor: remove unused upsert methods

We no longer depend on this library for ingestion, so we can remove all
this.

* refactor: Merge datahub_client into the main project

We no longer need to build this as a separate package, so we can lose
the separate pyproject.toml and combine the test suites.

* refactor: flatten datahub_client package structure

* build: remove redundant workflows

* fixup: fix dockerfile

.github/workflows/reusable-tests.yml

@@ -6,55 +6,6 @@ on:
   workflow_call:
 
 jobs:
-  datahub-client-path-filter:
-    runs-on: ubuntu-latest
-    outputs:
-      datahub-client: ${{ steps.changes.outputs.datahub-client }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          filters: |
-            datahub-client:
-              - 'lib/datahub-client/**'
-
-  datahub-client-tests:
-    name: Run datahub client tests
-    needs: datahub-client-path-filter
-    if: ${{ needs.datahub-client-path-filter.outputs.datahub-client == 'true' }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: lib/datahub-client
-    strategy:
-      fail-fast: false
-      max-parallel: 4
-      matrix:
-        python-version:
-          - "3.10"
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.4
-
-      - name: Install poetry
-        run: pipx install poetry
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: poetry
-          cache-dependency-path: lib/datahub-client/poetry.lock
-
-      - name: Poetry install
-        run: |
-          poetry install
-
-      - name: Run tests
-        run: |
-          poetry run pytest
-
   python-tests:
     name: Python tests
     runs-on: ubuntu-latest

Dockerfile

@@ -35,7 +35,6 @@ ENV POETRY_NO_INTERACTION=1 \
 
 # Install python dependencies to a virtualenv
 COPY pyproject.toml poetry.lock ./
-COPY lib ./lib
 RUN pip install poetry==1.8.4
 RUN poetry install --without dev --no-root && rm -rf $POETRY_CACHE_DIR
 
@@ -78,9 +77,10 @@ COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} manage.py ./
 COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} core ./core
 COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} users ./users
 COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} feedback ./feedback
+COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} datahub_client ./datahub_client
 COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} home ./home
 COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} templates ./templates
-COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} lib ./lib
+
 
 # Run django commands
 RUN python manage.py collectstatic --noinput

core/middleware.py

@@ -1,11 +1,12 @@
 import logging
 
-from data_platform_catalogue.client.exceptions import ConnectivityError
 from django.conf import settings
 from django.core.exceptions import BadRequest
 from django.http import Http404
 from django.shortcuts import render
 
+from datahub_client.exceptions import ConnectivityError
+
 logger = logging.getLogger(__name__)