From 7ce8d09aff074d7689a16b2db0e41fd1ced2ef9e Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 02:01:14 +0000 Subject: [PATCH 01/23] build(fix): change port for ui back to 5000 --- docker-compose.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4b74239c..6ea6a09c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,8 +22,7 @@ version: "3" services: # Database postgis: - image: postgis/postgis:15-3.3 - # image: arm64v8/postgres:15.3-alpine3.18 + image: postgis/postgis:${POSTGIS_TAG:-15-3.3-alpine} container_name: "underpass_postgis" ports: - "5439:5432" @@ -64,7 +63,7 @@ services: context: . dockerfile: docker/underpass-api.dockerfile # volumes: - # - ./python:/code/api + # - ./python:/code ports: - "8000:8000" networks: @@ -81,7 +80,7 @@ services: # volumes: # - js:/code ports: - - "5000:5000" + - "5000:8080" networks: internal: From 945f002f06666eac35784e4a19a0f1d76b63c283 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 02:02:12 +0000 Subject: [PATCH 02/23] build: update labels & use /code dir over /opt --- docker/underpass-api.dockerfile | 63 ++++++++++++++++++++++++++- docker/underpass-ui.dockerfile | 20 ++++++++- docker/underpass.dockerfile | 76 +++++++++++++++++++++++++++++++-- 3 files changed, 152 insertions(+), 7 deletions(-) diff --git a/docker/underpass-api.dockerfile b/docker/underpass-api.dockerfile index 541ef7b5..dbabe811 100644 --- a/docker/underpass-api.dockerfile +++ b/docker/underpass-api.dockerfile @@ -1,6 +1,24 @@ FROM python:3.9 -LABEL maintainer="Humanitarian OpenStreetMap Team" Description="This image provides the Underpass API" Vendor="HOT" Version="dev" +FROM docker.io/python:${PYTHON_TAG}-slim-bookworm as base +ARG APP_VERSION +ARG COMMIT_REF +LABEL org.hotosm.underpass.app-name="underpass-api" \ + org.hotosm.underpass.app-version="${APP_VERSION}" \ + org.hotosm.underpass.git-commit-ref="${COMMIT_REF:-none}" \ + org.hotosm.underpass.maintainer="sysadmin@hotosm.org" +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends "locales" "ca-certificates" \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && rm -rf /var/lib/apt/lists/* \ + && update-ca-certificates +# Set locale +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 WORKDIR /code @@ -14,7 +32,48 @@ COPY ./python/restapi /code/api/restapi RUN pip3 install -r /code/api/dbapi/requirements.txt RUN pip3 install -r /code/api/restapi/requirements.txt -WORKDIR /code/api/restapi +WORKDIR /opt/python +COPY python/dbapi/requirements.txt /opt/python/requirements.txt +COPY python/restapi/requirements.txt /opt/python/requirements2.txt +RUN pip install --user --no-warn-script-location --no-cache-dir \ + -r /opt/python/requirements.txt -r /opt/python/requirements2.txt + + + +FROM base as runtime +ARG PYTHON_TAG +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONFAULTHANDLER=1 \ + PATH="/home/appuser/.local/bin:$PATH" \ + PYTHONPATH="/opt/restapi" \ + PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_TAG/site-packages" \ + SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends \ + "postgresql-client" \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build \ + /root/.local \ + /home/appuser/.local + +COPY /python/dbapi /code/dbapi +COPY /python/restapi /code/restapi +WORKDIR /code/restapi + +# Add non-root user, permissions +RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \ + && chown -R appuser:appuser /code /home/appuser +# Change to non-root user +USER appuser +# Add Healthcheck +HEALTHCHECK --start-period=10s --interval=5s --retries=12 --timeout=5s \ + CMD curl --fail http://localhost:8000 || exit 1 ENTRYPOINT ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/underpass-ui.dockerfile b/docker/underpass-ui.dockerfile index 634750e1..c356db61 100644 --- a/docker/underpass-ui.dockerfile +++ b/docker/underpass-ui.dockerfile @@ -2,7 +2,25 @@ FROM node:alpine LABEL maintainer="Humanitarian OpenStreetMap Team" Description="This image provides the Underpass UI playground" Vendor="HOT" Version="dev" -RUN apk --no-cache add git +FROM docker.io/node:${NODE_TAG}-bookworm-slim as base +ARG APP_VERSION +ARG COMMIT_REF +LABEL org.hotosm.underpass.app-name="underpass-ui" \ + org.hotosm.underpass.app-version="${APP_VERSION}" \ + org.hotosm.underpass.git-commit-ref="${COMMIT_REF:-none}" \ + org.hotosm.underpass.maintainer="sysadmin@hotosm.org" +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends "locales" "ca-certificates" \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && rm -rf /var/lib/apt/lists/* \ + && update-ca-certificates +# Set locale +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 WORKDIR /code RUN git clone https://github.com/hotosm/underpass-ui.git . diff --git a/docker/underpass.dockerfile b/docker/underpass.dockerfile index d37f0b48..0d0c50e9 100644 --- a/docker/underpass.dockerfile +++ b/docker/underpass.dockerfile @@ -1,8 +1,53 @@ FROM ubuntu:latest -# This image is available as quay.io/hotosm/underpass:kinetic -LABEL maintainer="Humanitarian OpenStreetMap Team" Description="This image provides a build for Underpass" Vendor="HOT" Version="dev" -ENV DEBIAN_FRONTEND=noninteractive +FROM docker.io/debian:bookworm-slim as base +ARG APP_VERSION +ARG COMMIT_REF +LABEL org.hotosm.underpass.app-name="underpass" \ + org.hotosm.underpass.app-version="${APP_VERSION}" \ + org.hotosm.underpass.git-commit-ref="${COMMIT_REF:-none}" \ + org.hotosm.underpass.maintainer="sysadmin@hotosm.org" +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends "locales" "ca-certificates" \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && rm -rf /var/lib/apt/lists/* \ + && update-ca-certificates +# Set locale +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + + + +FROM base as build + +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends \ + "software-properties-common" \ + "libboost-all-dev" \ + "autotools-dev" \ + "swig" \ + "pkg-config" \ + "gcc" \ + "build-essential" \ + "ccache" \ + "dejagnu" \ + "libjemalloc-dev" \ + "libxml++2.6-dev" \ + "doxygen" \ + "libgdal-dev" \ + "libosmium2-dev" \ + "libpqxx-dev" \ + "postgresql" \ + "libgumbo-dev" \ + "librange-v3-dev" \ + "libtool" \ + && rm -rf /var/lib/apt/lists/* WORKDIR /code @@ -50,4 +95,27 @@ RUN ../configure && \ make -j $(nproc) && \ make install -WORKDIR /code + + + +FROM base as runtime +ENV PATH=$PATH:/code/build \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/code/build/.libs +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends \ + "libboost-all-dev" \ + "libgdal32" \ + "libxml++2.6-2v5" \ + "libjemalloc2" \ + "libpqxx-6.4" \ + "libgumbo1" \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /code/build /code/build +WORKDIR /code/build +# Add non-root user +RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser +# Change to non-root user +USER appuser +# ENTRYPOINT ["underpass"] From 1d7505e2e00d46feb4b599d42a2afc87e740bca0 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 02:14:00 +0000 Subject: [PATCH 03/23] build: add debug and prod stage to underpass dockerfile --- docker-compose.yml | 2 ++ docker/underpass.dockerfile | 23 ++++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6ea6a09c..9d2a0ffb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,10 +43,12 @@ services: # Underpass underpass: + image: "ghcr.io/hotosm/underpass/server:${APP_VERSION:-debug}" container_name: "underpass" build: context: . dockerfile: docker/underpass.dockerfile + target: debug environment: - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass command: tail -f /dev/null diff --git a/docker/underpass.dockerfile b/docker/underpass.dockerfile index 0d0c50e9..d283567b 100644 --- a/docker/underpass.dockerfile +++ b/docker/underpass.dockerfile @@ -22,7 +22,7 @@ ENV LC_ALL en_US.UTF-8 -FROM base as build +FROM base as deps RUN set -ex \ && apt-get update \ @@ -49,8 +49,10 @@ RUN set -ex \ "libtool" \ && rm -rf /var/lib/apt/lists/* -WORKDIR /code + +FROM deps as build +WORKDIR /code COPY ./src /code/src COPY ./config /code/config COPY ./setup /code/setup @@ -86,10 +88,8 @@ COPY ./dist /code/dist COPY ./docs /code/docs COPY ./ABOUT-NLS /code/ABOUT-NLS COPY ./config.rpath /code/config.rpath - WORKDIR /code RUN ./autogen.sh - WORKDIR /code/build RUN ../configure && \ make -j $(nproc) && \ @@ -98,7 +98,19 @@ RUN ../configure && \ -FROM base as runtime +FROM deps as debug +ENV PATH=$PATH:/code/build \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/code/build/.libs +COPY --from=build /code/build /code/build +WORKDIR /code/build +# Add non-root user +RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser +# Change to non-root user +USER appuser + + + +FROM base as prod ENV PATH=$PATH:/code/build \ LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/code/build/.libs RUN set -ex \ @@ -107,6 +119,7 @@ RUN set -ex \ -y --no-install-recommends \ "libboost-all-dev" \ "libgdal32" \ + "libosmium2-dev" \ "libxml++2.6-2v5" \ "libjemalloc2" \ "libpqxx-6.4" \ From a6a0b24a321ea3dcea5d8a1bed1c9ac1b88a58c1 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 02:17:59 +0000 Subject: [PATCH 04/23] build: allow port bind overrides in compose --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9d2a0ffb..9bf22480 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: image: postgis/postgis:${POSTGIS_TAG:-15-3.3-alpine} container_name: "underpass_postgis" ports: - - "5439:5432" + - "${DB_PORT:-5439}:5432" environment: - POSTGRES_DB=underpass - POSTGRES_USER=underpass @@ -67,7 +67,7 @@ services: # volumes: # - ./python:/code ports: - - "8000:8000" + - "${API_PORT:-8000}:8000" networks: internal: environment: @@ -82,7 +82,7 @@ services: # volumes: # - js:/code ports: - - "5000:8080" + - "${UI_PORT:-5000}:8080" networks: internal: From ee9cd55a9991477d94c0003b1b920496b982a7b1 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 03:16:48 +0000 Subject: [PATCH 05/23] ci(release): add workflow to build release images --- .github/workflows/release_img.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/release_img.yaml diff --git a/.github/workflows/release_img.yaml b/.github/workflows/release_img.yaml new file mode 100644 index 00000000..a48123b9 --- /dev/null +++ b/.github/workflows/release_img.yaml @@ -0,0 +1,29 @@ +name: ๐Ÿ”ง Build Release Images + +on: + release: + types: [published] + # Allow manual trigger + workflow_dispatch: + +jobs: + underpass-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + with: + dockerfile: docker/underpass.dockerfile + build_target: prod + image_name: ghcr.io/${{ github.repository }}/server + + api-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + with: + dockerfile: docker/underpass-api.dockerfile + build_target: runtime + image_name: ghcr.io/${{ github.repository }}/api + + ui-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + with: + dockerfile: docker/underpass-ui.dockerfile + build_target: runtime + image_name: ghcr.io/${{ github.repository }}/ui From 090bba188e84c2ecfe2fd626b8672cca3b3e8808 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 03:31:32 +0000 Subject: [PATCH 06/23] build: add ci stage in dockerfile, use for tests --- .dockerignore | 22 +++++++++++++++ docker-compose.yml | 3 ++- docker/ci-entrypoint.sh | 40 +++++++++++++++++++++++++++ docker/underpass.dockerfile | 54 +++++++++++-------------------------- 4 files changed, 79 insertions(+), 40 deletions(-) create mode 100644 .dockerignore create mode 100644 docker/ci-entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..2b9edd38 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Ignore everything +** + +# Allow files and directories + +# Underpass +!src +!config +!setup +!docker/bzip2.pc +!autogen.sh +!configure.ac +!Makefile.am +!m4 +!dist +!docs +!ABOUT-NLS +!config.rpath +!docker/ci-entrypoint.sh + +# API +!python diff --git a/docker-compose.yml b/docker-compose.yml index 9bf22480..de8dd10e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: build: context: . dockerfile: docker/underpass.dockerfile - target: debug + depends_on: [postgis] environment: - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass command: tail -f /dev/null @@ -64,6 +64,7 @@ services: build: context: . dockerfile: docker/underpass-api.dockerfile + depends_on: [underpass] # volumes: # - ./python:/code ports: diff --git a/docker/ci-entrypoint.sh b/docker/ci-entrypoint.sh new file mode 100644 index 00000000..8556f8cb --- /dev/null +++ b/docker/ci-entrypoint.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euo pipefail + +yellow_echo() { + local message="$1" + local separator="--------------------------------------------------------" + local sep_length=${#separator} + local pad_length=$(( (sep_length - ${#message}) / 2 )) + local pad="" + + for ((i=0; i Date: Thu, 9 Nov 2023 03:40:21 +0000 Subject: [PATCH 07/23] build: default to target: debug in local compose --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index de8dd10e..8818340a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,7 @@ services: build: context: . dockerfile: docker/underpass.dockerfile + target: debug depends_on: [postgis] environment: - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass From dd72e02711c14fa787c14412480da0819d14eb71 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 03:53:03 +0000 Subject: [PATCH 08/23] ci: rename release image file --> yml --- .github/workflows/{release_img.yaml => release_img.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{release_img.yaml => release_img.yml} (100%) diff --git a/.github/workflows/release_img.yaml b/.github/workflows/release_img.yml similarity index 100% rename from .github/workflows/release_img.yaml rename to .github/workflows/release_img.yml From 7b95f966ef4b4d26ca5449669600412326d6478c Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 03:53:30 +0000 Subject: [PATCH 09/23] ci: update ci_local.sh to use compose v2, simplify --- ci/ci_local.sh | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/ci/ci_local.sh b/ci/ci_local.sh index 0213d434..12c0dc4a 100755 --- a/ci/ci_local.sh +++ b/ci/ci_local.sh @@ -19,24 +19,12 @@ cd ${TEMP_DIR} make distclean -j `nproc` || true make clean -j `nproc` || true -DOCKER_DIR="${TEMP_DIR}" -DOCKER_COMPOSE_FILE="${DOCKER_DIR}/docker-compose.yml" -DOCKER_BASE_COMMAND="docker-compose -f ${DOCKER_COMPOSE_FILE}" +# Run CI +APP_VERSION=ci docker compose run underpass --exit-code-from=underpass -# Run the composition -${DOCKER_BASE_COMMAND} up -d +# Shut down containers +APP_VERSION=ci docker compose down -# Build Underpass Library and Binaries -${DOCKER_BASE_COMMAND} exec -T underpass sh -c "cd /code && git clean -fx && git clean -f -d && ./autogen.sh && (rm -rf build || true) && mkdir build && cd build && ../configure --enable-shared && make -j `nproc`" - -# Build and Run Underpass Tests - broken: alway succeeds -${DOCKER_BASE_COMMAND} exec -T underpass sh -c "cd /code/build/src/testsuite/libunderpass.all && make check -j `nproc`" - -# Comment the cleanup lines below or exit here if you want to run additional -# tests from a console in the temp container, for instance with: -${DOCKER_BASE_COMMAND} exec underpass bash - -${DOCKER_BASE_COMMAND} down echo "Remove temporary folder ${TEMP_DIR}" sudo rm -rf ${TEMP_DIR} From 7315bb091c0b8b4e67dab6d1db63f802c1049cf4 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 03:54:02 +0000 Subject: [PATCH 10/23] ci(tests): use reusable workflow test_compose for build & test --- .github/workflows/tests.yml | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9de44030..c0933460 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,29 +1,25 @@ -name: ๐Ÿงช Build and test +name: ๐Ÿงช Build and Test on: push: branches: - master + pull_request: + branches: + - master paths: - src/** - .github/workflows/** - docker/** jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - # Relies on the master branch built docker image for build-deps - - name: Start Docker Compose & build - run: cd docker && docker-compose up -d underpass - # Build and run tests - - name: Build and run tests - run: | - docker-compose -f docker-compose.yml exec -T underpass sh -c "cd /code/build && make check -j `nproc`" - exit_code=$? - if [ $exit_code -ne 0 ]; then - echo "Test failed with exit code $exit_code" - exit $exit_code - fi \ No newline at end of file + build-and-test: + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.2.1 + with: + image_name: ghcr.io/${{ github.repository }}/server + build_dockerfile: docker/underpass.dockerfile + compose_service: underpass + compose_command: echo "Tests complete." + # TODO update postgis image to use github repo var ${{ vars.POSTGIS_TAG }} + cache_extra_imgs: | + "docker.io/postgis/postgis:15-3.3-alpine" From cb540015784ced67bb91bb1f28711ffe20221b03 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 03:54:25 +0000 Subject: [PATCH 11/23] test: add act config for local gh-workflow testing --- .github/workflows/tests/pr_payload.json | 10 ++++++++++ .github/workflows/tests/push_payload.json | 4 ++++ .github/workflows/tests/test_ci.sh | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 .github/workflows/tests/pr_payload.json create mode 100644 .github/workflows/tests/push_payload.json create mode 100644 .github/workflows/tests/test_ci.sh diff --git a/.github/workflows/tests/pr_payload.json b/.github/workflows/tests/pr_payload.json new file mode 100644 index 00000000..265abb8a --- /dev/null +++ b/.github/workflows/tests/pr_payload.json @@ -0,0 +1,10 @@ +{ + "pull_request": { + "head": { + "ref": "feat/some-new-thing" + }, + "base": { + "ref": "master" + } + } +} diff --git a/.github/workflows/tests/push_payload.json b/.github/workflows/tests/push_payload.json new file mode 100644 index 00000000..07f80ac5 --- /dev/null +++ b/.github/workflows/tests/push_payload.json @@ -0,0 +1,4 @@ +{ + "base_ref ": "master", + "ref": "refs/heads/master" +} diff --git a/.github/workflows/tests/test_ci.sh b/.github/workflows/tests/test_ci.sh new file mode 100644 index 00000000..a7572d5c --- /dev/null +++ b/.github/workflows/tests/test_ci.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -e + +######################################## +# Note: run this from the repo root. +######################################## + +# Tests +act pull_request -W .github/workflows/tests.yml \ + -e .github/workflows/tests/pr_payload.json + +# Docs +act push -W .github/workflows/docs.yml \ + -e .github/workflows/tests/pr_payload.json + +# Release +act release -W .github/workflows/release_img.yml \ + -e .github/workflows/tests/push_payload.json From bc4bd3a78848a4a1319c32a17e010e9381307b96 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 12:55:42 +0000 Subject: [PATCH 12/23] build: add debug stages to dockerfiles for live reload --- docker-compose.yml | 14 ++++++++++++-- docker/underpass-api.dockerfile | 32 ++++++++++++++++++++++---------- docker/underpass-ui.dockerfile | 30 ++++++++++++++++++++++++++++-- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8818340a..647d6077 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,8 @@ services: context: . dockerfile: docker/underpass.dockerfile target: debug + args: + APP_VERSION: ${APP_VERSION:-debug} depends_on: [postgis] environment: - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass @@ -60,11 +62,15 @@ services: internal: # Underpass API - underpass-api: + api: + image: "ghcr.io/hotosm/underpass/api:${APP_VERSION:-debug}" container_name: "underpass_api" build: context: . dockerfile: docker/underpass-api.dockerfile + target: debug + args: + APP_VERSION: ${APP_VERSION:-debug} depends_on: [underpass] # volumes: # - ./python:/code @@ -76,11 +82,15 @@ services: - UNDERPASS_API_DB=postgresql://underpass:underpass@postgis/underpass # Underpass UI - underpass-ui: + ui: + image: "ghcr.io/hotosm/underpass/ui:${APP_VERSION:-debug}" container_name: "underpass_ui" build: context: . dockerfile: docker/underpass-ui.dockerfile + target: debug + args: + APP_VERSION: ${APP_VERSION:-debug} # volumes: # - js:/code ports: diff --git a/docker/underpass-api.dockerfile b/docker/underpass-api.dockerfile index dbabe811..54fbb7a2 100644 --- a/docker/underpass-api.dockerfile +++ b/docker/underpass-api.dockerfile @@ -26,12 +26,14 @@ RUN apt-get update && apt-get -y install \ postgresql \ libpq-dev -COPY ./python/dbapi /code/api/dbapi -COPY ./python/restapi /code/api/restapi - -RUN pip3 install -r /code/api/dbapi/requirements.txt -RUN pip3 install -r /code/api/restapi/requirements.txt - +FROM base as build +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends \ + "build-essential" \ + "libpq-dev" \ + && rm -rf /var/lib/apt/lists/* WORKDIR /opt/python COPY python/dbapi/requirements.txt /opt/python/requirements.txt COPY python/restapi/requirements.txt /opt/python/requirements2.txt @@ -57,15 +59,12 @@ RUN set -ex \ -y --no-install-recommends \ "postgresql-client" \ && rm -rf /var/lib/apt/lists/* - COPY --from=build \ /root/.local \ /home/appuser/.local - COPY /python/dbapi /code/dbapi COPY /python/restapi /code/restapi WORKDIR /code/restapi - # Add non-root user, permissions RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \ && chown -R appuser:appuser /code /home/appuser @@ -75,5 +74,18 @@ USER appuser HEALTHCHECK --start-period=10s --interval=5s --retries=12 --timeout=5s \ CMD curl --fail http://localhost:8000 || exit 1 -ENTRYPOINT ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] + +FROM runtime as debug +CMD ["uvicorn", "app.main:api", \ + "--host", "0.0.0.0", "--port", "8000", \ + "--reload", "--log-level", "critical", "--no-access-log"] + + + +FROM runtime as prod +# Pre-compile packages to .pyc (init speed gains) +RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)" +# Note: 4 uvicorn workers as running with docker, change to 1 worker for Kubernetes +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", \ + "--workers", "4", "--log-level", "critical", "--no-access-log"] diff --git a/docker/underpass-ui.dockerfile b/docker/underpass-ui.dockerfile index c356db61..b275f352 100644 --- a/docker/underpass-ui.dockerfile +++ b/docker/underpass-ui.dockerfile @@ -22,7 +22,16 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -WORKDIR /code + + +FROM base as deps +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends \ + "git" \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /repo RUN git clone https://github.com/hotosm/underpass-ui.git . RUN yarn install RUN yarn build @@ -30,4 +39,21 @@ RUN yarn build WORKDIR /code/playground RUN yarn install -ENTRYPOINT ["yarn", "cosmos"] + + +FROM deps as build +RUN yarn run cosmos:export + + + +FROM deps as debug +CMD yarn run cosmos + + + +FROM docker.io/devforth/spa-to-http:1.0.3 as prod +WORKDIR /app +# Add non-root user, permissions +RUN adduser -D -u 1001 -h /home/appuser appuser +USER appuser +COPY --from=build --chown=appuser:appuser /repo/playground/cosmos-export/ . From 1558e7750243973f4ee03660ed79324e3dd907f8 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 12:55:57 +0000 Subject: [PATCH 13/23] ci: build release imgs to prod docker target --- .github/workflows/release_img.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_img.yml b/.github/workflows/release_img.yml index a48123b9..268de593 100644 --- a/.github/workflows/release_img.yml +++ b/.github/workflows/release_img.yml @@ -18,12 +18,12 @@ jobs: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 with: dockerfile: docker/underpass-api.dockerfile - build_target: runtime + build_target: prod image_name: ghcr.io/${{ github.repository }}/api ui-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 with: dockerfile: docker/underpass-ui.dockerfile - build_target: runtime + build_target: prod image_name: ghcr.io/${{ github.repository }}/ui From 883040d9a38482bdd95c064db6d55e360be5f6a9 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 13:08:03 +0000 Subject: [PATCH 14/23] build: allow bind mounting code for debug images --- docker-compose.yml | 8 +++++--- docker/underpass-api.dockerfile | 2 +- docker/underpass-ui.dockerfile | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 647d6077..825675d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,8 +72,8 @@ services: args: APP_VERSION: ${APP_VERSION:-debug} depends_on: [underpass] - # volumes: - # - ./python:/code + volumes: + - ./python:/code ports: - "${API_PORT:-8000}:8000" networks: @@ -91,8 +91,10 @@ services: target: debug args: APP_VERSION: ${APP_VERSION:-debug} + # # Mount underpass-ui repo # volumes: - # - js:/code + # - ../underpass-ui/src:/code/src + # - ../underpass-ui/playground:/code/playground ports: - "${UI_PORT:-5000}:8080" networks: diff --git a/docker/underpass-api.dockerfile b/docker/underpass-api.dockerfile index 54fbb7a2..6f5cbff1 100644 --- a/docker/underpass-api.dockerfile +++ b/docker/underpass-api.dockerfile @@ -77,7 +77,7 @@ HEALTHCHECK --start-period=10s --interval=5s --retries=12 --timeout=5s \ FROM runtime as debug -CMD ["uvicorn", "app.main:api", \ +CMD ["uvicorn", "main:app", \ "--host", "0.0.0.0", "--port", "8000", \ "--reload", "--log-level", "critical", "--no-access-log"] diff --git a/docker/underpass-ui.dockerfile b/docker/underpass-ui.dockerfile index b275f352..5587aaac 100644 --- a/docker/underpass-ui.dockerfile +++ b/docker/underpass-ui.dockerfile @@ -31,11 +31,11 @@ RUN set -ex \ -y --no-install-recommends \ "git" \ && rm -rf /var/lib/apt/lists/* -WORKDIR /repo +WORKDIR /code RUN git clone https://github.com/hotosm/underpass-ui.git . RUN yarn install RUN yarn build - +# Cosmos WORKDIR /code/playground RUN yarn install From ed9f68f6ecaf09d07d2139b4692c3a40f64f09dd Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 13:14:56 +0000 Subject: [PATCH 15/23] build: rename underpass/server image --> underpass --- .github/workflows/release_img.yml | 2 +- .github/workflows/tests.yml | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_img.yml b/.github/workflows/release_img.yml index 268de593..dcf5ff67 100644 --- a/.github/workflows/release_img.yml +++ b/.github/workflows/release_img.yml @@ -12,7 +12,7 @@ jobs: with: dockerfile: docker/underpass.dockerfile build_target: prod - image_name: ghcr.io/${{ github.repository }}/server + image_name: ghcr.io/${{ github.repository }} api-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0933460..d05dd75b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: build-and-test: uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.2.1 with: - image_name: ghcr.io/${{ github.repository }}/server + image_name: ghcr.io/${{ github.repository }} build_dockerfile: docker/underpass.dockerfile compose_service: underpass compose_command: echo "Tests complete." diff --git a/docker-compose.yml b/docker-compose.yml index 825675d3..7971d54a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: # Underpass underpass: - image: "ghcr.io/hotosm/underpass/server:${APP_VERSION:-debug}" + image: "ghcr.io/hotosm/underpass:${APP_VERSION:-debug}" container_name: "underpass" build: context: . From cb23f63fd651c0663e8306460569ed697307543a Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 9 Nov 2023 13:22:50 +0000 Subject: [PATCH 16/23] build(fix): underpass-ui image change build dir --> /code --- docker/underpass-ui.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/underpass-ui.dockerfile b/docker/underpass-ui.dockerfile index 5587aaac..ee82459d 100644 --- a/docker/underpass-ui.dockerfile +++ b/docker/underpass-ui.dockerfile @@ -56,4 +56,4 @@ WORKDIR /app # Add non-root user, permissions RUN adduser -D -u 1001 -h /home/appuser appuser USER appuser -COPY --from=build --chown=appuser:appuser /repo/playground/cosmos-export/ . +COPY --from=build --chown=appuser:appuser /code/playground/cosmos-export/ . From 429122282b644fe5ab8cd84aa785cb53f9e367be Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 21 Nov 2023 12:38:38 +0000 Subject: [PATCH 17/23] fix: rebase conflicts for multistage dockerfiles --- docker/underpass-api.dockerfile | 3 ++- docker/underpass-ui.dockerfile | 3 +-- docker/underpass.dockerfile | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docker/underpass-api.dockerfile b/docker/underpass-api.dockerfile index 6f5cbff1..01e27370 100644 --- a/docker/underpass-api.dockerfile +++ b/docker/underpass-api.dockerfile @@ -1,4 +1,5 @@ -FROM python:3.9 +ARG PYTHON_TAG=${PYTHON_TAG:-3.10} + FROM docker.io/python:${PYTHON_TAG}-slim-bookworm as base ARG APP_VERSION diff --git a/docker/underpass-ui.dockerfile b/docker/underpass-ui.dockerfile index ee82459d..10fdb43b 100644 --- a/docker/underpass-ui.dockerfile +++ b/docker/underpass-ui.dockerfile @@ -1,6 +1,5 @@ -FROM node:alpine +ARG NODE_TAG=${NODE_TAG:-18} -LABEL maintainer="Humanitarian OpenStreetMap Team" Description="This image provides the Underpass UI playground" Vendor="HOT" Version="dev" FROM docker.io/node:${NODE_TAG}-bookworm-slim as base ARG APP_VERSION diff --git a/docker/underpass.dockerfile b/docker/underpass.dockerfile index 5ede474d..af0cef12 100644 --- a/docker/underpass.dockerfile +++ b/docker/underpass.dockerfile @@ -1,5 +1,3 @@ -FROM ubuntu:latest - FROM docker.io/debian:bookworm-slim as base ARG APP_VERSION ARG COMMIT_REF From 809fc4f37ebf26bd87afbd21a2c1909f28511811 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 21 Nov 2023 12:42:12 +0000 Subject: [PATCH 18/23] ci: update gh-workflow versions --> 1.3.0 --- .github/workflows/release_img.yml | 6 +++--- .github/workflows/tests.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_img.yml b/.github/workflows/release_img.yml index dcf5ff67..9dc3fdd6 100644 --- a/.github/workflows/release_img.yml +++ b/.github/workflows/release_img.yml @@ -8,21 +8,21 @@ on: jobs: underpass-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.3.0 with: dockerfile: docker/underpass.dockerfile build_target: prod image_name: ghcr.io/${{ github.repository }} api-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.3.0 with: dockerfile: docker/underpass-api.dockerfile build_target: prod image_name: ghcr.io/${{ github.repository }}/api ui-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.3.0 with: dockerfile: docker/underpass-ui.dockerfile build_target: prod diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d05dd75b..dbf126aa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: jobs: build-and-test: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.2.1 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.3.0 with: image_name: ghcr.io/${{ github.repository }} build_dockerfile: docker/underpass.dockerfile From 29e2446b504b2777f11f49dd5e841cb0470fbc5b Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 21 Nov 2023 13:36:13 +0000 Subject: [PATCH 19/23] ci: rename build release img workflow --- .github/workflows/release_img.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_img.yml b/.github/workflows/release_img.yml index 9dc3fdd6..8607b387 100644 --- a/.github/workflows/release_img.yml +++ b/.github/workflows/release_img.yml @@ -1,4 +1,4 @@ -name: ๐Ÿ”ง Build Release Images +name: ๐Ÿ”ง Build Release Image on: release: From 356cd3306de67e94da3a80950a75a02dda2f8a3c Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 30 Nov 2023 07:34:32 +0000 Subject: [PATCH 20/23] ci: update workflow versions 1.3.0 --> 1.4.0 --- .github/workflows/release_img.yml | 6 +++--- .github/workflows/tests.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_img.yml b/.github/workflows/release_img.yml index 8607b387..f88d0de9 100644 --- a/.github/workflows/release_img.yml +++ b/.github/workflows/release_img.yml @@ -8,21 +8,21 @@ on: jobs: underpass-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.3.0 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.0 with: dockerfile: docker/underpass.dockerfile build_target: prod image_name: ghcr.io/${{ github.repository }} api-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.3.0 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.0 with: dockerfile: docker/underpass-api.dockerfile build_target: prod image_name: ghcr.io/${{ github.repository }}/api ui-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.3.0 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.0 with: dockerfile: docker/underpass-ui.dockerfile build_target: prod diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dbf126aa..623cce1d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: jobs: build-and-test: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.3.0 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.0 with: image_name: ghcr.io/${{ github.repository }} build_dockerfile: docker/underpass.dockerfile From b8b45937a4fcf3db03e0c8631dda4be6e7155b65 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 30 Nov 2023 12:20:45 +0000 Subject: [PATCH 21/23] build: fix underpass-ui compose port mapping --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7971d54a..33c28035 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,7 +96,7 @@ services: # - ../underpass-ui/src:/code/src # - ../underpass-ui/playground:/code/playground ports: - - "${UI_PORT:-5000}:8080" + - "${UI_PORT:-8080}:5000" networks: internal: From e217b4461fed10d36dfd0588bf383ad6bc5d0edd Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 30 Nov 2023 12:21:20 +0000 Subject: [PATCH 22/23] ci: update workflows 1.4.0 --> 1.4.1 --- .github/workflows/release_img.yml | 6 +++--- .github/workflows/tests.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_img.yml b/.github/workflows/release_img.yml index f88d0de9..c8dd2168 100644 --- a/.github/workflows/release_img.yml +++ b/.github/workflows/release_img.yml @@ -8,21 +8,21 @@ on: jobs: underpass-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.0 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.1 with: dockerfile: docker/underpass.dockerfile build_target: prod image_name: ghcr.io/${{ github.repository }} api-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.0 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.1 with: dockerfile: docker/underpass-api.dockerfile build_target: prod image_name: ghcr.io/${{ github.repository }}/api ui-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.0 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.1 with: dockerfile: docker/underpass-ui.dockerfile build_target: prod diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 623cce1d..ec5535cc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: jobs: build-and-test: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.0 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.1 with: image_name: ghcr.io/${{ github.repository }} build_dockerfile: docker/underpass.dockerfile From 06c131d5b8abc0871ef79c3ef4001b01c08bacd3 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 1 Dec 2023 09:10:45 +0000 Subject: [PATCH 23/23] ci: allow manual trigger of test workflow --- .github/workflows/docs.yml | 4 +++- .github/workflows/tests.yml | 2 ++ docker-compose.yml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index cf518cef..06d250f5 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,9 +6,11 @@ on: - docs/** - mkdocs.yml branches: - - master + - master + permissions: contents: write + jobs: deploy: runs-on: ubuntu-latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ec5535cc..d86713f8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,8 @@ on: - src/** - .github/workflows/** - docker/** + # Allow manual trigger + workflow_dispatch: jobs: build-and-test: diff --git a/docker-compose.yml b/docker-compose.yml index 33c28035..e0a74e16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,7 +57,7 @@ services: command: tail -f /dev/null volumes: - ./replication:/code/build/replication - # - ./:/code + # - ./:/code networks: internal: