Introduce poppy-cli into the tegola container

docker-compose.yml

@@ -23,6 +23,9 @@ services:
     volumes:
       - ${APP_CODE_PATH_HOST}:${APP_CODE_PATH_CONTAINER}
       - ${DATA_PATH_HOST}/pbf:${APP_CODE_PATH_CONTAINER}/pbf
+      - ./workspace/mapping.yml:/etc/imposm/mapping.yml
+      - ./workspace/osm-initial-import.sh:/usr/bin/osm-initial-import
+
     extra_hosts:
       - "dockerhost:${DOCKER_HOST_IP}"
     tty: true
@@ -49,11 +52,11 @@ services:
       - postgres-postgis
       - cassandra
       - redis
-      - tegola
     volumes:
       - ${APP_CODE_PATH_HOST}/kartotherian:/home/kartotherian
       - ${APP_CODE_PATH_HOST}/dependencies:/srv/dependencies
-      - ${APP_CODE_PATH_HOST}/kartosm/expiretiles:/srv/expiretiles
+      - ${DATA_PATH_HOST}/imposm:/srv/imposm
+      - ./kartotherian/sources.docker.yaml:/etc/opt/sources.docker.yaml
     tty: true
     networks:
       - frontend
@@ -63,7 +66,7 @@ services:
   postgres-postgis:
     image: postgis/postgis:11-3.1
     volumes:
-      - ${DATA_PATH_HOST}/imposm:/var/lib/postgresql/data
+      - ${DATA_PATH_HOST}/postgres:/var/lib/postgresql/data
     ports:
       - "${PGPORT}:5432"
     environment:
@@ -113,9 +116,17 @@ services:
 
 ### Tegola ##################################
   tegola:
-    build: https://github.com/go-spatial/tegola.git#v0.14.x
+    build: ./tegola
+#    build: https://github.com/go-spatial/tegola.git#v0.14.x
+    environment:
+      - TEGOLA_BROKER_URL=redis://redis:6379
+      - TEGOLA_QUEUE_NAME=pregen
+      - TEGOLA_PATH=/opt/tegola
+      - TEGOLA_CONFIG_PATH=/etc/tegola.toml
     volumes:
       - ./tegola/tegola.toml:/etc/tegola.toml
+      - ./tegola/pregenerate-maps-tile.sh:/etc/pregenerate-maps-tile.sh
+      - ${DATA_PATH_HOST}/tegola/cache:/etc/cache
     ports:
       - "${TEGOLA_PORT}:8080"
     networks:

kartotherian/Dockerfile

@@ -35,7 +35,7 @@ RUN apt-get update && apt-get install -y nodejs \
 ENV NVM_DIR $HOME/.nvm
 
 # Avoid using cached layers from this point to allow easy changing of nvm version install
-ARG CACHEBUST=1 
+ARG CACHEBUST=1
 
 RUN wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.6/install.sh | bash \
     && . $NVM_DIR/nvm.sh && nvm install 10.15.2
@@ -47,7 +47,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends yarn
 
 COPY config.tilerator.docker.template.yaml /etc/opt/config.tilerator.docker.template.yaml
 COPY config.kartotherian.docker.template.yaml /etc/opt/config.kartotherian.docker.template.yaml
-COPY sources.docker.yaml /etc/opt/sources.docker.yaml
 COPY generate_config.sh /usr/local/bin/generate_config
 RUN chmod +x /usr/local/bin/generate_config
 

kartotherian/config.kartotherian.docker.template.yaml

@@ -87,6 +87,7 @@ services:
       - "@wikimedia/kartotherian-overzoom"
       - "@wikimedia/kartotherian-postgres"
       - "@wikimedia/kartotherian-substantial"
+      - "@wikimedia/kartotherian-tilelive-switch"
       - "@kartotherian/tilelive-tmsource"
       - "@kartotherian/tilelive-vector"
       - "@mapbox/tilejson"

kartotherian/sources.docker.yaml

@@ -15,11 +15,6 @@ genraw:
       port: { var: osmdb-port }
       user: {var: osmdb-user}
       password: {var: osmdb-pswd}
-# expand name_ attributes into individual tags
-gen:
-  uri: json2tags://
-  params:
-    source: {ref: genraw}
 
 # Tegola backend
 http-tegola:
@@ -38,6 +33,49 @@ http-tegola:
       - id: poi_label
       - id: road_label
 
+# Tegola beta-cluster backend
+beta-cluster-http-tegola:
+  uri: https://tegola.wmflabs.org/maps/osm/{z}/{x}/{y}.pbf
+  setInfo:
+    vector_layers:
+      - id: landuse
+      - id: waterway
+      - id: water
+      - id: aeroway
+      - id: building
+      - id: road
+      - id: admin
+      - id: country_label
+      - id: place_label
+      - id: poi_label
+      - id: road_label
+
+# expand name_ attributes into individual tags
+gen-tegola:
+  uri: json2tags://
+  params:
+#    source: {ref: beta-cluster-http-tegola}
+    source: {ref: http-tegola}
+
+# Mirror requests to a secondary source
+switch:
+  uri: mirror://
+  params:
+    source: {ref: genraw}
+    secondarySource: { ref: beta-cluster-http-tegola }
+    enableMirror: true
+    loadBalancer:
+      bucketSize: 1000 # milliseconds
+      totalBuckets: 10 # number of buckets in the histogram
+      threshold: 0.5 # % of load to the mirrored source
+
+# expand name_ attributes into individual tags
+gen:
+  uri: json2tags://
+  params:
+    source: {ref: switch}
+
+
 # Cassandra storage
 v4:
   public: true
@@ -82,119 +120,15 @@ genview:
     Pragma: no-cache
     Expires: 0
 
-# Filter out any small tiles without significant data on or after zoom 10
-substgen:
-  public: true
-  formats: [pbf]
-  uri: substantial://
-  params:
-    source: {ref: gen}
-    minzoom: 10
-    minsize: 500
-    layers: [water, landuse]
-
-# If tile has been filtered out by substgen, use overzoom to recreate it
-substgenoz:
-  public: true
-  formats: [pbf]
-  uri: overzoom://
-  params:
-    source: {ref: substgen}
-
-# View tiles from generator, without the insignificant ones (will show blank squares)
-substgenview:
-  public: true
-  formats: [png,json,headers,svg,jpeg]
-  scales: [1.3, 1.5, 2, 2.6, 3]
-  static: true
-  maxheight: 2048
-  maxwidth: 2048
-  uri: tmstyle://
-  yaml:
-    npm: ["@kartotherian/osm-bright-style", "project.yml"]
-  yamlSetParams:
-    source: {ref: substgen}
-
-# View tiles from generator, and use overzoom for any filtered out ones
-substgenozview:
-  public: true
-  formats: [png,json,headers,svg,jpeg]
-  scales: [1.3, 1.5, 2, 2.6, 3]
-  static: true
-  maxheight: 2048
-  maxwidth: 2048
-  uri: tmstyle://
-  yaml:
-    npm: ["@kartotherian/osm-bright-style", "project.yml"]
-  yamlSetParams:
-    source: {ref: substgenoz}
-
-# Filter out any tiles stored in the v4 that are not significant
-substv4:
-  public: true
-  formats: [pbf]
-  uri: substantial://
-  params:
-    source: {ref: v4}
-    minzoom: 10
-    minsize: 500
-    layers: [water, landuse]
-
-# Use overzoom on v4 tiles for any filtered out insignificant tiles
-substv4oz:
-  public: true
-  formats: [pbf]
-  uri: overzoom://
-  params:
-    source: {ref: substv4}
-
-# Show v4 tiles with insignificant ones showing as blank squares
-substv4view:
-  public: true
-  formats: [png,json,headers,svg,jpeg]
-  scales: [1.3, 1.5, 2, 2.6, 3]
-  static: true
-  maxheight: 2048
-  maxwidth: 2048
-  uri: tmstyle://
-  yaml:
-    npm: ["@kartotherian/osm-bright-style", "project.yml"]
-  yamlSetParams:
-    source: {ref: substv4}
-
-# Show v4 tiles and use overzoom for those that have been filtered out
-substv4ozview:
-  public: true
-  formats: [png,json,headers,svg,jpeg]
-  scales: [1.3, 1.5, 2, 2.6, 3]
-  static: true
-  maxheight: 2048
-  maxwidth: 2048
-  uri: tmstyle://
-  yaml:
-    npm: ["@kartotherian/osm-bright-style", "project.yml"]
-  yamlSetParams:
-    source: {ref: substv4oz}
-
 # Default OSM PBF source
 osm-pbf:
   public: true
   formats: [pbf]
   uri: overzoom://
   params:
-    source: {ref: v4}
-    maxzoom: 19
-  overrideInfo:
-    attribution: 'Map data © <a href="http://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
-    tiles: ["http://localhost:6533/osm-pbf/{z}/{x}/{y}.pbf"]
-
-# Default OSM PBF source
-osm-pbf-tegola:
-  public: true
-  formats: [pbf]
-  uri: overzoom://
-  params:
-    source: {ref: http-tegola}
+    source: {ref: gen-tegola}
+#    source: {ref: beta-cluster-http-tegola}
+#    source: { ref: v4 }
     maxzoom: 19
   overrideInfo:
     attribution: 'Map data © <a href="http://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
@@ -208,38 +142,7 @@ babel:
     combineName: false
     defaultLanguage: local
 
-babel-tegola:
-  uri: babel://
-  params:
-    source: {ref: osm-pbf-tegola}
-    tag: name
-    combineName: false
-    defaultLanguage: local
-
-# OSM map with international labeling - tegola backed
-osm-intl-tegola:
-  public: true
-  formats: [png,json,headers,svg,jpeg]
-  scales: [1.3, 1.5, 2, 2.6, 3]
-  maxzoom: 19
-  static: true
-  maxheight: 2048
-  maxwidth: 2048
-  uri: tmstyle://
-  yaml:
-    npm: ["@kartotherian/osm-bright-style", "project.yml"]
-  yamlSetParams:
-    source: {ref: babel-tegola}
-  overrideInfo:
-    attribution: 'Map data © <a href="http://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
-    tiles: ["http://localhost:6533/osm-intl-tegola/{z}/{x}/{y}.png"]
-  defaultHeaders:
-    Cache-Control: 'no-cache, no-store, must-revalidate'
-    Pragma: no-cache
-    Expires: 0
-
-
-# OSM map with international labeling - will be used as default
+# OSM map with international labeling
 osm-intl:
   public: true
   formats: [png,json,headers,svg,jpeg]

makefile

@@ -11,9 +11,6 @@ osm:
 generate_config:
 	docker-compose exec kartotherian generate_config
 
-generate_config_osm2pgsql:
-	docker-compose exec kartotherian-osm2pgsql generate_config
-
 # Cassandra Keyspace setup https://wikitech.wikimedia.org/wiki/Maps/Keyspace_Setup
 keyspace_setup:
 	docker-compose exec kartotherian bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/tilerator/scripts/tileshell.js --config /etc/opt/config.tilerator.docker.yaml --source /etc/opt/sources.docker.yaml"
@@ -24,12 +21,6 @@ run_tilerator:
 run_kartotherian:
 	docker-compose exec kartotherian bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/kartotherian/server.js -c /etc/opt/config.kartotherian.docker.yaml"
 
-run_tilerator_osm2pgsql:
-	docker-compose exec kartotherian-osm2pgsql bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/tilerator/server.js -c /etc/opt/config.tilerator.docker.yaml"
-
-run_kartotherian_osm2pgsql:
-	docker-compose exec kartotherian-osm2pgsql bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/kartotherian/server.js -c /etc/opt/config.kartotherian.docker.yaml"
-
 npm_test:
 	docker-compose exec kartotherian bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && npm test"
 
@@ -48,11 +39,16 @@ clean:
 	docker-compose exec kartotherian bash -c "./clean_node_modules.sh"
 
 imposm_run:
-	docker-compose exec workspace bash -c "imposm run -config /srv/kartosm/config.json" -expiretiles-zoom 15
+	docker-compose exec workspace bash -c "imposm run -config /etc/imposm/config.json" -expiretiles-zoom 15
 
 notify_tilerator:
 	docker-compose exec kartotherian bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/tilerator/scripts/tileshell.js  --config /etc/opt/config.tilerator.docker.yaml -j.fromZoom 10 -j.beforeZoom 16 -j.generatorId gen -j.storageId v4 -j.deleteEmpty -j.expdirpath /srv/expiretiles -j.expmask '(expire\.list\.*)|(\.tiles)' -j.statefile /home/kartotherian/expire.state"
 
+pregen_dequeue:
+    docker-compose exec tegola /etc/pregenerate-maps-tile.sh
+pregen_enqueue:
+    docker-compose exec tegola poppy --broker-url redis://redis:6379 --queue-name pregen enqueue --message-input tile 1/1/1
+
 install:
 	# TODO
 	# Check if kartotherian is installed and clone if it isn't

tegola/Dockerfile

@@ -0,0 +1,38 @@
+# Based on upstream Dockerfile https://github.com/go-spatial/tegola/blob/v0.14.x/Dockerfile
+FROM golang:1.16.2-alpine3.12 AS build
+
+ARG VERSION="Version Not Set"
+ENV VERSION="${VERSION}"
+
+# Only needed for CGO support at time of build, results in no noticable change in binary size
+# incurs approximately 1:30 extra build time (1:54 vs 0:27) to install packages.  Doesn't impact
+# development as these layers are drawn from cache after the first build.
+RUN apk update \
+	&& apk add musl-dev=1.1.24-r10 \
+	&& apk add gcc=9.3.0-r2 \
+	&& apk add git
+
+# Set up source for compilation
+RUN mkdir -p /go/src/github.com/go-spatial/tegola
+RUN git clone "https://gerrit.wikimedia.org/r/operations/software/tegola" /go/src/github.com/go-spatial/tegola
+
+# Build binary
+RUN cd /go/src/github.com/go-spatial/tegola/cmd/tegola \
+ 	&& go build -v -ldflags "-w -X 'github.com/go-spatial/tegola/cmd/tegola/cmd.Version=${VERSION}'" -gcflags "-N -l" -o /opt/tegola \
+	&& chmod a+x /opt/tegola
+
+# Create minimal deployment image, just alpine & the binary
+FROM python:3-alpine3.12
+
+RUN apk update \
+	&& apk add ca-certificates \
+	&& apk add git \
+	&& rm -rf /var/cache/apk/*
+
+RUN pip install "git+https://github.com/wikimedia/poppy-cli@dequeue-until-empty" \
+    redis \
+    kafka-python
+
+COPY --from=build /opt/tegola /opt/
+WORKDIR /opt
+ENTRYPOINT ["/opt/tegola"]