Tiler software updates

.gitignore

@@ -36,4 +36,5 @@ overpass-api-db/
 data/*/
 !data/README.md
 images/changeset-replication-job/config.yaml
-values_copy.yaml
+values_copy.yaml
+images/tiler-server/tegola-cache/

compose/tiler.yml

@@ -1,4 +1,4 @@
-version: '3.8'
+# version: '3.8'
 services:
   # ######################################################
   # ### Tiler DB
@@ -8,61 +8,53 @@ services:
     build:
       context: ../images/tiler-db
       dockerfile: Dockerfile
-    # ports:
-    # - "5433:5432"
+    ports:
+    - "5432:5432"
     volumes:
     - ../data/tiler-db-data:/var/lib/postgresql/data
     env_file:
     - ../envs/.env.tiler-db
     restart: always
-  # ######################################################
-  # ### Tiler imposm for importing data
-  # ######################################################
-  tiler-imposm:
-    image: osmseed-tiler-imposm:v1
-    build:
-      context: ../images/tiler-imposm
-      dockerfile: Dockerfile
-    volumes:
-    - ../data/tiler-imposm-data:/mnt/data
-    # - ../images/tiler-imposm:/osm
-    depends_on:
-    - tiler-db
-    command:
-    - sh
-    - -c
-    - "sleep 60 && ./start.sh"
-    env_file:
-    - ../envs/.env.tiler-db
-    - ../envs/.env.tiler-imposm
-    healthcheck:
-      test: ["CMD", "./liveness.sh"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-    restart: always
-  # ######################################################
-  # ### Tiler server
-  # ######################################################
+  # # ######################################################
+  # # ### Tiler imposm for importing data
+  # # ######################################################
+  # tiler-imposm:
+  #   image: osmseed-tiler-imposm:v1
+  #   build:
+  #     context: ../images/tiler-imposm
+  #     dockerfile: Dockerfile
+  #   volumes:
+  #   - ../data/tiler-imposm-data:/mnt/data
+  #   # - ../images/tiler-imposm:/osm
+  #   command:
+  #   - sh
+  #   - -c
+  #   - "./start.sh"
+  #   env_file:
+  #   - ../envs/.env.tiler-db
+  #   - ../envs/.env.tiler-imposm
+  #   healthcheck:
+  #     test: ["CMD", "./liveness.sh"]
+  #     interval: 30s
+  #     timeout: 10s
+  #     retries: 3
+  #   restart: always
+  # # ######################################################
+  # # ### Tiler server
+  # # ######################################################
   tiler-server:
     image: osmseed-tiler-server:v1
+    # platform: linux/amd64
     build:
       context: ../images/tiler-server
       dockerfile: Dockerfile
     volumes:
-    - ../data/tiler-server-data:/mnt/data
-    - ../data/tiler-imposm-data:/mnt/data/imposm
-    depends_on:
-    - tiler-db
-    - tiler-imposm
+      - ../images/tiler-server:/app
+      - ../data/tiler-imposm-data:/mnt/data/imposm
     ports:
-    - "9090:9090"
-    command:
-    - sh
-    - -c
-    - "./start.sh & ./expire-watcher.sh"
+      - "9090:9090"
+    command: "./start.sh"
     env_file:
-    - ../envs/.env.tiler-db
-    - ../envs/.env.tiler-server
-    restart: always
-
+      - ../envs/.env.tiler-db
+      - ../envs/.env.tiler-server
+    restart: always

envs/.env.tiler-imposm.example

@@ -7,4 +7,6 @@ TILER_IMPORT_FROM=osm
 TILER_IMPORT_PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf
 SEQUENCE_NUMBER=4768
 REPLICATION_URL=https://download.openstreetmap.fr/replication/europe/monaco/minute/
-OVERWRITE_STATE=false
+OVERWRITE_STATE=false
+IMPORT_NATURAL_EARTH=true
+IMPORT_OSM_LAND=true

images/tiler-db/Dockerfile

@@ -1,11 +1,7 @@
-FROM mdillon/postgis:9.5
-RUN rm /etc/apt/sources.list.d/pgdg.list
-RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list
-RUN sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list 
-RUN sed -i '/stretch-updates/d' /etc/apt/sources.list 
-RUN apt-get update && apt-get -y install apt-transport-https
-RUN echo "deb https://apt-archive.postgresql.org/pub/repos/apt/ stretch-pgdg main 9.5" /etc/apt/sources.list.d/pgdg.list
-RUN apt-get update && apt-get install -y git ca-certificates
+FROM postgis/postgis:14-3.4
+RUN apt-get update
+RUN apt-get install -y git ca-certificates
+
 COPY ./config/docker-entrypoint.sh /usr/local/bin/
 RUN mkdir -p /docker-entrypoint-initdb.d
 COPY ./config/initdb_db.sh /docker-entrypoint-initdb.d/postgis.sh

images/tiler-db/config/initdb_db.sh

@@ -1,20 +1,10 @@
 #!/bin/sh
 set -e
 
-# Create the 'template_postgis' template db
-"${psql[@]}" <<- 'EOSQL'
-CREATE DATABASE template_postgis;
-UPDATE pg_database SET datistemplate = TRUE WHERE datname = 'template_postgis';
-EOSQL
-
-# Load PostGIS into both template_database and $POSTGRES_DB
-for DB in template_postgis "$POSTGRES_DB"; do
-	echo "Loading PostGIS extensions into $DB"
-	"${psql[@]}" --dbname="$DB" <<-'EOSQL'
-		CREATE EXTENSION IF NOT EXISTS postgis;
-		CREATE EXTENSION IF NOT EXISTS postgis_topology;
-		CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
-		CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;
-		CREATE EXTENSION IF NOT EXISTS hstore;
-EOSQL
-done
+# Add hstore into the DB
+for DB in template_postgis "$POSTGRES_DB" "${@}"; do
+    echo "Updating extensions '$DB'"
+    psql --dbname="$DB" -c "
+        CREATE EXTENSION IF NOT EXISTS hstore;
+    "
+done

images/tiler-server/Dockerfile

@@ -1,57 +1,41 @@
-FROM golang:1.14.1-alpine3.11 AS build
-ENV VERSION="v0.8.1"
-RUN apk update
-RUN apk add musl-dev=1.1.24-r3 \
-	gcc \
-    bash \
-    git \
-    postgresql \
-    postgresql-contrib
+FROM gospatial/tegola:v0.20.0
 
-RUN mkdir -p /go/src/github.com/go-spatial/tegola
-RUN git clone https://github.com/go-spatial/tegola.git /go/src/github.com/go-spatial/tegola
-RUN cd /go/src/github.com/go-spatial/tegola && git checkout v0.13.0
-RUN cd /go/src/github.com/go-spatial/tegola/cmd/tegola \
-	&& go build -gcflags "-N -l" -o /opt/tegola \ 
-	&& chmod a+x /opt/tegola
-RUN ln -s /opt/tegola /usr/bin/tegola
-
-RUN apk add --update \
-    python3 \
-    py-pip \
-    py-cffi \
-    py-cryptography \
-  && pip install --upgrade pip \
-  && apk add --virtual build-deps \
+RUN apk --no-cache add \
     gcc \
     libffi-dev \
-    python3-dev \
+    python3 \
+    py3-pip \
+    py3-cffi \
+    py3-cryptography \
     linux-headers \
     musl-dev \
     openssl-dev \
-    curl
+    bash \
+    git \
+    postgresql-client \
+    coreutils \
+    jq
 
-# Install aws-cli and  gsutil
-RUN pip3 install awscli
-RUN curl -sSL https://sdk.cloud.google.com | bash
-RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
+RUN pip install --upgrade pip && pip3 install awscli mercantile 
 
-RUN pip3 install mercantile \
-    && apk del build-deps \
-    && rm -rf /var/cache/apk/* \
-    && apk --purge -v del py-pip
 
-RUN apk add --update coreutils jq && rm -rf /var/cache/apk/*
+RUN ln -s /opt/tegola /usr/bin/tegola
+
+COPY ./config  /opt/config/
+COPY build_config.py  /opt/
+RUN mkdir /opt/tegola_config
+RUN cd /opt/ && python build_config.py \
+    --template config/config.template.toml \
+    --providers config/providers \
+    --output /opt/tegola_config/config.toml
 
-# Volumen
-VOLUME /mnt/data
 # Copy config and  exec files
-COPY ./config/config.toml  /opt/tegola_config/config.toml
 COPY ./tile2bounds.py .
 COPY ./start.sh .
 COPY ./expire-watcher.sh .
 COPY ./seed-by-diffs.sh .
 COPY ./tile_cache_downloader.sh .
 COPY ./rm_tegola_ps.sh .
 
+ENTRYPOINT ["/bin/bash", "-c"]
 CMD ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh & ./rm_tegola_ps.sh

images/tiler-server/build_config.py

@@ -0,0 +1,38 @@
+import os
+import argparse
+parser = argparse.ArgumentParser(description='Merge TOML files into a configuration file.')
+parser.add_argument('--template', default='config/config.template.toml', help='Path to the configuration template file.')
+parser.add_argument('--providers', default='config/providers', help='Directory containing provider TOML files.')
+parser.add_argument('--output', default='config/config.toml', help='Output configuration file path.')
+args = parser.parse_args()
+
+config_template_file = args.template
+providers_dir = args.providers
+output_file_path = args.output
+toml_files = [file for file in os.listdir(providers_dir) if file.endswith(".toml")]
+
+# Read TOML files
+new_configs = {}
+for toml_file in toml_files:
+    dir_toml_file = os.path.join(providers_dir, toml_file)
+    with open(dir_toml_file, "r") as file:
+        new_configs[dir_toml_file] = file.read()
+
+with open(config_template_file, "r") as main_file:
+    content = main_file.read()
+
+# Replace the content of main.toml with the content read from other TOML files
+for toml_file, toml_file_content in new_configs.items():
+    print(f"Copy {toml_file} to config.toml")
+    section_header = "[['{}']]".format(toml_file.replace("config/", ""))
+    indentation_level = content.find(section_header)
+    if indentation_level != -1:
+        # Find the appropriate number of tabs or spaces for indentation
+        preceding_newline = content.rfind('\n', 0, indentation_level)
+        indentation = content[preceding_newline + 1:indentation_level]
+        toml_file_content = f"###### From {toml_file} \n" + toml_file_content
+        new_values=toml_file_content.replace("\n", "\n" + indentation)
+        content = content.replace(section_header, new_values)
+
+with open(output_file_path, "w") as output_file:
+    output_file.write(content)