Merge pull request #444 from spwoodcock/build/multi-stage-dockerfiles

Optimise dockerfiles, fix ci, release images

.dockerignore

@@ -0,0 +1,22 @@
+# Ignore everything
+**
+
+# Allow files and directories
+
+# Underpass
+!src
+!config
+!setup
+!docker/bzip2.pc
+!autogen.sh
+!configure.ac
+!Makefile.am
+!m4
+!dist
+!docs
+!ABOUT-NLS
+!config.rpath
+!docker/ci-entrypoint.sh
+
+# API
+!python

.github/workflows/docs.yml

@@ -6,9 +6,11 @@ on:
       - docs/**
       - mkdocs.yml
     branches:
-      - master 
+      - master
+
 permissions:
   contents: write
+
 jobs:
   deploy:
     runs-on: ubuntu-latest

.github/workflows/release_img.yml

@@ -0,0 +1,29 @@
+name: 🔧 Build Release Image
+
+on:
+  release:
+    types: [published]
+  # Allow manual trigger
+  workflow_dispatch:
+
+jobs:
+  underpass-build:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      dockerfile: docker/underpass.dockerfile
+      build_target: prod
+      image_name: ghcr.io/${{ github.repository }}
+
+  api-build:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      dockerfile: docker/underpass-api.dockerfile
+      build_target: prod
+      image_name: ghcr.io/${{ github.repository }}/api
+
+  ui-build:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      dockerfile: docker/underpass-ui.dockerfile
+      build_target: prod
+      image_name: ghcr.io/${{ github.repository }}/ui

.github/workflows/tests.yml

@@ -1,29 +1,27 @@
-name: 🧪 Build and test
+name: 🧪 Build and Test
 
 on:
   push:
     branches:
       - master
+  pull_request:
+    branches:
+      - master
     paths:
       - src/**
       - .github/workflows/**
       - docker/**
+  # Allow manual trigger
+  workflow_dispatch:
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      # Relies on the master branch built docker image for build-deps
-      - name: Start Docker Compose & build
-        run: cd docker && docker-compose up -d underpass
-      # Build and run tests
-      - name: Build and run tests 
-        run: |
-          docker-compose -f docker-compose.yml exec -T underpass sh -c "cd /code/build && make check -j `nproc`"
-          exit_code=$?
-          if [ $exit_code -ne 0 ]; then
-            echo "Test failed with exit code $exit_code"
-            exit $exit_code
-          fi
+  build-and-test:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      image_name: ghcr.io/${{ github.repository }}
+      build_dockerfile: docker/underpass.dockerfile
+      compose_service: underpass
+      compose_command: echo "Tests complete."
+      # TODO update postgis image to use github repo var ${{ vars.POSTGIS_TAG }}
+      cache_extra_imgs: |
+        "docker.io/postgis/postgis:15-3.3-alpine"

.github/workflows/tests/pr_payload.json

@@ -0,0 +1,10 @@
+{
+  "pull_request": {
+    "head": {
+      "ref": "feat/some-new-thing"
+    },
+    "base": {
+      "ref": "master"
+    }
+  }
+}

.github/workflows/tests/push_payload.json

@@ -0,0 +1,4 @@
+{
+  "base_ref ": "master",
+  "ref": "refs/heads/master"
+}

.github/workflows/tests/test_ci.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+########################################
+# Note: run this from the repo root.
+########################################
+
+# Tests
+act pull_request -W .github/workflows/tests.yml \
+    -e .github/workflows/tests/pr_payload.json
+
+# Docs
+act push -W .github/workflows/docs.yml \
+    -e .github/workflows/tests/pr_payload.json
+
+# Release
+act release -W .github/workflows/release_img.yml \
+    -e .github/workflows/tests/push_payload.json

ci/ci_local.sh

@@ -19,24 +19,12 @@ cd ${TEMP_DIR}
 make distclean -j `nproc` || true
 make clean -j `nproc` || true
 
-DOCKER_DIR="${TEMP_DIR}"
-DOCKER_COMPOSE_FILE="${DOCKER_DIR}/docker-compose.yml"
-DOCKER_BASE_COMMAND="docker-compose -f ${DOCKER_COMPOSE_FILE}"
+# Run CI
+APP_VERSION=ci docker compose run underpass --exit-code-from=underpass
 
-# Run the composition
-${DOCKER_BASE_COMMAND} up -d
+# Shut down containers
+APP_VERSION=ci docker compose down
 
-# Build Underpass Library and Binaries
-${DOCKER_BASE_COMMAND} exec -T underpass sh -c "cd /code && git clean -fx && git clean -f -d && ./autogen.sh && (rm -rf build || true) && mkdir build && cd build && ../configure --enable-shared && make -j `nproc`"
-
-# Build and Run Underpass Tests - broken: alway succeeds
-${DOCKER_BASE_COMMAND} exec -T underpass sh -c "cd /code/build/src/testsuite/libunderpass.all && make check -j `nproc`"
-
-# Comment the cleanup lines below or exit here if you want to run additional
-# tests from a console in the temp container, for instance with:
-${DOCKER_BASE_COMMAND} exec underpass bash
-
-${DOCKER_BASE_COMMAND} down
 echo "Remove temporary folder ${TEMP_DIR}"
 sudo rm -rf ${TEMP_DIR}
 

docker-compose.yml

@@ -22,11 +22,10 @@ version: "3"
 services:
   # Database
   postgis:
-    image: postgis/postgis:15-3.3
-    # image: arm64v8/postgres:15.3-alpine3.18
+    image: postgis/postgis:${POSTGIS_TAG:-15-3.3-alpine}
     container_name: "underpass_postgis"
     ports:
-      - "5439:5432"
+      - "${DB_PORT:-5439}:5432"
     environment:
       - POSTGRES_DB=underpass
       - POSTGRES_USER=underpass
@@ -44,44 +43,60 @@ services:
 
   # Underpass
   underpass:
+    image: "ghcr.io/hotosm/underpass:${APP_VERSION:-debug}"
     container_name: "underpass"
     build:
      context: .
      dockerfile: docker/underpass.dockerfile
+     target: debug
+     args:
+        APP_VERSION: ${APP_VERSION:-debug}
+    depends_on: [postgis]
     environment:
       - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass
     command: tail -f /dev/null
     volumes:
       - ./replication:/code/build/replication
-  #   - ./:/code
+      # - ./:/code
     networks:
       internal:
 
   # Underpass API
-  underpass-api:
+  api:
+    image: "ghcr.io/hotosm/underpass/api:${APP_VERSION:-debug}"
     container_name: "underpass_api"
     build:
      context: .
      dockerfile: docker/underpass-api.dockerfile
-    # volumes:
-    #   - ./python:/code/api
+     target: debug
+     args:
+        APP_VERSION: ${APP_VERSION:-debug}
+    depends_on: [underpass]
+    volumes:
+      - ./python:/code
     ports:
-      - "8000:8000"
+      - "${API_PORT:-8000}:8000"
     networks:
       internal:
     environment:
       - UNDERPASS_API_DB=postgresql://underpass:underpass@postgis/underpass
 
   # Underpass UI
-  underpass-ui:
+  ui:
+    image: "ghcr.io/hotosm/underpass/ui:${APP_VERSION:-debug}"
     container_name: "underpass_ui"
     build:
      context: .
      dockerfile: docker/underpass-ui.dockerfile
+     target: debug
+     args:
+        APP_VERSION: ${APP_VERSION:-debug}
+    # # Mount underpass-ui repo
     # volumes:
-    #   - js:/code
+    #   - ../underpass-ui/src:/code/src
+    #   - ../underpass-ui/playground:/code/playground
     ports:
-      - "5000:5000"
+      - "${UI_PORT:-8080}:5000"
     networks:
       internal:
 

docker/ci-entrypoint.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+yellow_echo() {
+    local message="$1"
+    local separator="--------------------------------------------------------"
+    local sep_length=${#separator}
+    local pad_length=$(( (sep_length - ${#message}) / 2 ))
+    local pad=""
+
+    for ((i=0; i<pad_length; i++)); do
+        pad="$pad "
+    done
+
+    echo ""
+    echo -e "\e[0;33m$separator\e[0m"
+    echo -e "\e[0;33m$pad$message$pad\e[0m"
+    echo -e "\e[0;33m$separator\e[0m"
+    echo ""
+}
+
+# Generate makefiles
+yellow_echo "Generating Makefiles"
+./autogen.sh
+
+echo
+echo "Entering 'build' directory"
+cd build
+
+# Test build works
+yellow_echo "Building Underpass"
+../configure
+make -j $(nproc)
+make install
+
+# Run tests
+yellow_echo "Running Tests"
+cd /code/build/src/testsuite/libunderpass.all
+make check -j `nproc`