Merge pull request #449 from hotosm/revert-447-feature/ci

Revert "Merge: Optimise dockerfiles, fix ci, release images"

.github/workflows/tests.yml

@@ -1,25 +1,29 @@
-name: 🧪 Build and Test
+name: 🧪 Build and test
 
 on:
   push:
     branches:
       - master
-  pull_request:
-    branches:
-      - master
     paths:
       - src/**
       - .github/workflows/**
       - docker/**
 
 jobs:
-  build-and-test:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected]
-    with:
-      image_name: ghcr.io/${{ github.repository }}
-      build_dockerfile: docker/underpass.dockerfile
-      compose_service: underpass
-      compose_command: echo "Tests complete."
-      # TODO update postgis image to use github repo var ${{ vars.POSTGIS_TAG }}
-      cache_extra_imgs: |
-        "docker.io/postgis/postgis:15-3.3-alpine"
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      # Relies on the master branch built docker image for build-deps
+      - name: Start Docker Compose & build
+        run: cd docker && docker-compose up -d underpass
+      # Build and run tests
+      - name: Build and run tests 
+        run: |
+          docker-compose -f docker-compose.yml exec -T underpass sh -c "cd /code/build && make check -j `nproc`"
+          exit_code=$?
+          if [ $exit_code -ne 0 ]; then
+            echo "Test failed with exit code $exit_code"
+            exit $exit_code
+          fi

ci/ci_local.sh

@@ -19,12 +19,24 @@ cd ${TEMP_DIR}
 make distclean -j `nproc` || true
 make clean -j `nproc` || true
 
-# Run CI
-APP_VERSION=ci docker compose run underpass --exit-code-from=underpass
+DOCKER_DIR="${TEMP_DIR}"
+DOCKER_COMPOSE_FILE="${DOCKER_DIR}/docker-compose.yml"
+DOCKER_BASE_COMMAND="docker-compose -f ${DOCKER_COMPOSE_FILE}"
 
-# Shut down containers
-APP_VERSION=ci docker compose down
+# Run the composition
+${DOCKER_BASE_COMMAND} up -d
 
+# Build Underpass Library and Binaries
+${DOCKER_BASE_COMMAND} exec -T underpass sh -c "cd /code && git clean -fx && git clean -f -d && ./autogen.sh && (rm -rf build || true) && mkdir build && cd build && ../configure --enable-shared && make -j `nproc`"
+
+# Build and Run Underpass Tests - broken: alway succeeds
+${DOCKER_BASE_COMMAND} exec -T underpass sh -c "cd /code/build/src/testsuite/libunderpass.all && make check -j `nproc`"
+
+# Comment the cleanup lines below or exit here if you want to run additional
+# tests from a console in the temp container, for instance with:
+${DOCKER_BASE_COMMAND} exec underpass bash
+
+${DOCKER_BASE_COMMAND} down
 echo "Remove temporary folder ${TEMP_DIR}"
 sudo rm -rf ${TEMP_DIR}
 

docker-compose.yml

@@ -22,10 +22,11 @@ version: "3"
 services:
   # Database
   postgis:
-    image: postgis/postgis:${POSTGIS_TAG:-15-3.3-alpine}
+    image: postgis/postgis:15-3.3
+    # image: arm64v8/postgres:15.3-alpine3.18
     container_name: "underpass_postgis"
     ports:
-      - "${DB_PORT:-5439}:5432"
+      - "5439:5432"
     environment:
       - POSTGRES_DB=underpass
       - POSTGRES_USER=underpass
@@ -43,61 +44,44 @@ services:
 
   # Underpass
   underpass:
-    image: "ghcr.io/hotosm/underpass:${APP_VERSION:-debug}"
-    container_name: underpass
+    container_name: "underpass"
     build:
      context: .
      dockerfile: docker/underpass.dockerfile
-     target: debug
-     args:
-        APP_VERSION: ${APP_VERSION:-debug}
-    depends_on: [postgis]
     environment:
       - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass
     command: tail -f /dev/null
     volumes:
       - ./replication:/code/build/replication
-  # # Mount code
   #   - ./:/code
     networks:
       internal:
 
   # Underpass API
-  api:
-    image: "ghcr.io/hotosm/underpass/api:${APP_VERSION:-debug}"
-    container_name: underpass_api
+  underpass-api:
+    container_name: "underpass_api"
     build:
      context: .
      dockerfile: docker/underpass-api.dockerfile
-     target: debug
-     args:
-        APP_VERSION: ${APP_VERSION:-debug}
-    depends_on: [underpass]
-    volumes:
-      - ./python:/code
+    # volumes:
+    #   - ./python:/code/api
     ports:
-      - "${API_PORT:-8000}:8000"
+      - "8000:8000"
     networks:
       internal:
     environment:
       - UNDERPASS_API_DB=postgresql://underpass:underpass@postgis/underpass
 
   # Underpass UI
-  ui:
-    image: "ghcr.io/hotosm/underpass/ui:${APP_VERSION:-debug}"
-    container_name: underpass_ui
+  underpass-ui:
+    container_name: "underpass_ui"
     build:
      context: .
      dockerfile: docker/underpass-ui.dockerfile
-     target: debug
-     args:
-        APP_VERSION: ${APP_VERSION:-debug}
-    # # Mount underpass-ui repo
     # volumes:
-    #   - ../underpass-ui/src:/code/src
-    #   - ../underpass-ui/playground:/code/playground
+    #   - js:/code
     ports:
-      - "${UI_PORT:-5000}:5000"
+      - "5000:5000"
     networks:
       internal:
 

docker/underpass-api.dockerfile

@@ -1,85 +1,20 @@
 FROM python:3.9
 
-FROM docker.io/python:${PYTHON_TAG}-slim-bookworm as base
-ARG APP_VERSION
-ARG COMMIT_REF
-LABEL org.hotosm.underpass.app-name="underpass-api" \
-      org.hotosm.underpass.app-version="${APP_VERSION}" \
-      org.hotosm.underpass.git-commit-ref="${COMMIT_REF:-none}" \
-      org.hotosm.underpass.maintainer="[email protected]"
-RUN set -ex \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install \
-    -y --no-install-recommends "locales" "ca-certificates" \
-    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
-    && rm -rf /var/lib/apt/lists/* \
-    && update-ca-certificates
-# Set locale
-RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+LABEL maintainer="Humanitarian OpenStreetMap Team" Description="This image provides the Underpass API" Vendor="HOT" Version="dev"
 
 WORKDIR /code
 
 RUN apt-get update && apt-get -y install \ 
     postgresql \
     libpq-dev
 
-FROM base as build
-RUN set -ex \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install \
-    -y --no-install-recommends \
-        "build-essential" \
-        "libpq-dev" \
-    && rm -rf /var/lib/apt/lists/*
-WORKDIR /opt/python
-COPY python/dbapi/requirements.txt /opt/python/requirements.txt
-COPY python/restapi/requirements.txt /opt/python/requirements2.txt
-RUN pip install --user --no-warn-script-location --no-cache-dir \
-    -r /opt/python/requirements.txt -r /opt/python/requirements2.txt
+COPY ./python/dbapi /code/api/dbapi
+COPY ./python/restapi /code/api/restapi
 
-FROM base as runtime
-ARG PYTHON_TAG
-ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONFAULTHANDLER=1 \
-    PATH="/home/appuser/.local/bin:$PATH" \
-    PYTHONPATH="/opt/restapi" \
-    PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_TAG/site-packages" \
-    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
-    REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
-    CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
-RUN set -ex \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install \
-    -y --no-install-recommends \
-        "postgresql-client" \
-    && rm -rf /var/lib/apt/lists/*
-COPY --from=build \
-    /root/.local \
-    /home/appuser/.local
-COPY /python/dbapi /code/dbapi
-COPY /python/restapi /code/restapi
-WORKDIR /code/restapi
-# Add non-root user, permissions
-RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \
-    && chown -R appuser:appuser /code /home/appuser
-# Change to non-root user
-USER appuser
-# Add Healthcheck
-HEALTHCHECK --start-period=10s --interval=5s --retries=12 --timeout=5s \
-    CMD curl --fail http://localhost:8000 || exit 1
+RUN pip3 install -r /code/api/dbapi/requirements.txt
+RUN pip3 install -r /code/api/restapi/requirements.txt
 
-FROM runtime as debug
-CMD ["uvicorn", "main:app", \
-    "--host", "0.0.0.0", "--port", "8000", \
-    "--reload", "--log-level", "critical", "--no-access-log"]
+WORKDIR /code/api/restapi
+
+ENTRYPOINT ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
 
-FROM runtime as prod
-# Pre-compile packages to .pyc (init speed gains)
-RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
-# Note: 4 uvicorn workers as running with docker, change to 1 worker for Kubernetes
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", \
-    "--workers", "4", "--log-level", "critical", "--no-access-log"]