Merge pull request #303 from headwaymaps/mirk/landcover

Add vector landcover to terrain for low zoom
headwaymaps · Jan 18, 2024 · 050647c · 050647c
2 parents adb1a5c + 7d0a554
commit 050647c
Show file tree

Hide file tree

Showing 16 changed files with 166 additions and 93 deletions.
diff --git a/.env-example b/.env-example
@@ -36,4 +36,4 @@ HEADWAY_HTTP_PORT=8080
 export HEADWAY_S3_BUCKET=""
 
 # Where artifacts are downloaded from by k8s init-containers
-export HEADWAY_K8S_ARTIFACT_ROOT="https://data.example.com/${HEADWAY_S3_BUCKET}"
+export HEADWAY_K8S_ARTIFACT_ROOT="https://data.example.com${HEADWAY_S3_BUCKET}"
diff --git a/Earthfile b/Earthfile
@@ -41,7 +41,7 @@ save:
     BUILD +save-elasticsearch --area=${area} --countries=${countries}
     BUILD +save-placeholder --area=${area} --countries=${countries}
     BUILD +save-pelias-config --area=${area} --countries=${countries}
-    BUILD +save-tileserver-natural-earth
+    BUILD +save-tileserver-terrain
 
 save-polylines:
     FROM +save-base
@@ -168,10 +168,12 @@ save-pelias-config:
     COPY (+pelias-config/pelias.json --area=${area} --countries=${countries}) /pelias.json
     SAVE ARTIFACT /pelias.json AS LOCAL ./data/${area}.pelias.json
 
-save-tileserver-natural-earth:
+save-tileserver-terrain:
     FROM +downloader-base
-    RUN wget -nv https://publicdata.ellenhp.workers.dev/natural_earth_2_shaded_relief.raster.mbtiles
-    SAVE ARTIFACT natural_earth_2_shaded_relief.raster.mbtiles AS LOCAL ./data/natural_earth.mbtiles
+    RUN wget -nv https://data.maps.earth/terrain.mbtiles
+    SAVE ARTIFACT terrain.mbtiles AS LOCAL ./data/terrain.mbtiles
+    RUN wget -nv https://data.maps.earth/landcover.mbtiles
+    SAVE ARTIFACT landcover.mbtiles AS LOCAL ./data/landcover.mbtiles
 
 images:
     FROM debian:bookworm-slim

diff --git a/bin/publish-data b/bin/publish-data
@@ -86,7 +86,8 @@ upload "${HEADWAY_AREA}*.graph.obj.zst" "${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}
 upload "${HEADWAY_AREA}*.gtfs.tar.zst"  "${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}"
 
 # These files are generic across all areas
-upload natural_earth.mbtiles "${HEADWAY_DATA_TAG}"
+upload terrain.mbtiles "${HEADWAY_DATA_TAG}"
+upload landcover.mbtiles "${HEADWAY_DATA_TAG}"
 
 rets=()
 for pid in ${upload_pids[*]}; do

diff --git a/docker-compose-with-transit.yaml b/docker-compose-with-transit.yaml
@@ -4,10 +4,12 @@ services:
     image: ghcr.io/headwaymaps/tileserver-init:latest
     env_file: .env
     environment:
-      MBTILES_ARTIFACT_DEST_PATH: /data/${HEADWAY_AREA}.mbtiles
-      MBTILES_ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.mbtiles
-      NATURAL_EARTH_ARTIFACT_DEST_PATH: /data/natural_earth.mbtiles
-      NATURAL_EARTH_ARTIFACT_SOURCE_PATH: /bootstrap/natural_earth.mbtiles
+      AREAMAP_ARTIFACT_DEST: /data/${HEADWAY_AREA}.mbtiles
+      AREAMAP_ARTIFACT_SOURCE: /bootstrap/${HEADWAY_AREA}.mbtiles
+      TERRAIN_ARTIFACT_DEST_PATH: /data/terrain.mbtiles
+      TERRAIN_ARTIFACT_SOURCE_PATH: /bootstrap/terrain.mbtiles
+      LANDCOVER_ARTIFACT_DEST: /data/landcover.mbtiles
+      LANDCOVER_ARTIFACT_SOURCE: /bootstrap/landcover.mbtiles
     volumes:
       - "./data/:/bootstrap/:ro"
       - "tileserver_data:/data/:rw"

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -4,10 +4,12 @@ services:
     image: ghcr.io/headwaymaps/tileserver-init:latest
     env_file: .env
     environment:
-      MBTILES_ARTIFACT_DEST_PATH: /data/${HEADWAY_AREA}.mbtiles
-      MBTILES_ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.mbtiles
-      NATURAL_EARTH_ARTIFACT_DEST_PATH: /data/natural_earth.mbtiles
-      NATURAL_EARTH_ARTIFACT_SOURCE_PATH: /bootstrap/natural_earth.mbtiles
+      AREAMAP_ARTIFACT_DEST: /data/${HEADWAY_AREA}.mbtiles
+      AREAMAP_ARTIFACT_SOURCE: /bootstrap/${HEADWAY_AREA}.mbtiles
+      TERRAIN_ARTIFACT_DEST: /data/terrain.mbtiles
+      TERRAIN_ARTIFACT_SOURCE: /bootstrap/terrain.mbtiles
+      LANDCOVER_ARTIFACT_DEST: /data/landcover.mbtiles
+      LANDCOVER_ARTIFACT_SOURCE: /bootstrap/landcover.mbtiles
     volumes:
       - "./data/:/bootstrap/:ro"
       - "tileserver_data:/data/:rw"

diff --git a/k8s/_template/deployment-config.yaml.tpl b/k8s/_template/deployment-config.yaml.tpl
@@ -7,8 +7,9 @@ data:
   public-url: ${HEADWAY_PUBLIC_URL}
   bbox: "${HEADWAY_BBOX}"
   enable-transit-routing: "${HEADWAY_ENABLE_TRANSIT_ROUTING}"
-  natural-earth-source-url: ${HEADWAY_K8S_ARTIFACT_ROOT}/${HEADWAY_DATA_TAG}/natural_earth.mbtiles
-  mbtiles-source-url: ${HEADWAY_K8S_ARTIFACT_ROOT}/${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}/${HEADWAY_AREA}.mbtiles
+  terrain-source-url: ${HEADWAY_K8S_ARTIFACT_ROOT}/${HEADWAY_DATA_TAG}/terrain.mbtiles
+  landcover-source-url: ${HEADWAY_K8S_ARTIFACT_ROOT}/${HEADWAY_DATA_TAG}/landcover.mbtiles
+  areamap-source-url: ${HEADWAY_K8S_ARTIFACT_ROOT}/${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}/${HEADWAY_AREA}.mbtiles
   valhalla-artifact-url: ${HEADWAY_K8S_ARTIFACT_ROOT}/${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}/${HEADWAY_AREA}.valhalla.tar.zst
   placeholder-artifact-url: ${HEADWAY_K8S_ARTIFACT_ROOT}/${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}/${HEADWAY_AREA}.placeholder.tar.zst
   elasticsearch-artifact-url: ${HEADWAY_K8S_ARTIFACT_ROOT}/${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}/${HEADWAY_AREA}.elasticsearch.tar.zst${OTP_GRAPHS_YAML}

diff --git a/k8s/_template/tileserver-deployment.yaml.tpl b/k8s/_template/tileserver-deployment.yaml.tpl
@@ -22,20 +22,27 @@ spec:
           env:
             - name: HEADWAY_AREA
               value: area
-            - name: NATURAL_EARTH_ARTIFACT_DEST_PATH
-              value: /data/natural_earth.mbtiles
-            - name: NATURAL_EARTH_ARTIFACT_URL
+            - name: TERRAIN_ARTIFACT_DEST
+              value: /data/terrain.mbtiles
+            - name: TERRAIN_ARTIFACT_SOURCE
               valueFrom:
                 configMapKeyRef:
                   name: deployment-config
-                  key: natural-earth-source-url
-            - name: MBTILES_ARTIFACT_DEST_PATH
+                  key: terrain-source-url
+            - name: LANDCOVER_ARTIFACT_DEST
+              value: /data/landcover.mbtiles
+            - name: LANDCOVER_ARTIFACT_SOURCE
+              valueFrom:
+                configMapKeyRef:
+                  name: deployment-config
+                  key: landcover-source-url
+            - name: AREAMAP_ARTIFACT_DEST
               value: /data/area.mbtiles
-            - name: MBTILES_ARTIFACT_URL
+            - name: AREAMAP_ARTIFACT_SOURCE
               valueFrom:
                 configMapKeyRef:
                   name: deployment-config
-                  key: mbtiles-source-url
+                  key: areamap-source-url
           resources:
             limits:
               memory: 200Mi

diff --git a/k8s/configs/planet-dev/deployment-config.yaml b/k8s/configs/planet-dev/deployment-config.yaml
@@ -7,8 +7,9 @@ data:
   public-url: https://maps.earth
   bbox: ""
   enable-transit-routing: "1"
-  natural-earth-source-url: https://data.example.com/0.6.0/natural_earth.mbtiles
-  mbtiles-source-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.mbtiles
+  terrain-source-url: https://data.example.com/0.6.0/terrain.mbtiles
+  landcover-source-url: https://data.example.com/0.6.0/landcover.mbtiles
+  areamap-source-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.mbtiles
   valhalla-artifact-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.valhalla.tar.zst
   placeholder-artifact-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.placeholder.tar.zst
   elasticsearch-artifact-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.elasticsearch.tar.zst

diff --git a/k8s/configs/planet-dev/tileserver-deployment.yaml b/k8s/configs/planet-dev/tileserver-deployment.yaml
@@ -22,20 +22,27 @@ spec:
           env:
             - name: HEADWAY_AREA
               value: area
-            - name: NATURAL_EARTH_ARTIFACT_DEST_PATH
-              value: /data/natural_earth.mbtiles
-            - name: NATURAL_EARTH_ARTIFACT_URL
+            - name: TERRAIN_ARTIFACT_DEST
+              value: /data/terrain.mbtiles
+            - name: TERRAIN_ARTIFACT_SOURCE
               valueFrom:
                 configMapKeyRef:
                   name: deployment-config
-                  key: natural-earth-source-url
-            - name: MBTILES_ARTIFACT_DEST_PATH
+                  key: terrain-source-url
+            - name: LANDCOVER_ARTIFACT_DEST
+              value: /data/landcover.mbtiles
+            - name: LANDCOVER_ARTIFACT_SOURCE
+              valueFrom:
+                configMapKeyRef:
+                  name: deployment-config
+                  key: landcover-source-url
+            - name: AREAMAP_ARTIFACT_DEST
               value: /data/area.mbtiles
-            - name: MBTILES_ARTIFACT_URL
+            - name: AREAMAP_ARTIFACT_SOURCE
               valueFrom:
                 configMapKeyRef:
                   name: deployment-config
-                  key: mbtiles-source-url
+                  key: areamap-source-url
           resources:
             limits:
               memory: 200Mi

diff --git a/k8s/configs/planet/deployment-config.yaml b/k8s/configs/planet/deployment-config.yaml
@@ -7,8 +7,9 @@ data:
   public-url: https://maps.earth
   bbox: ""
   enable-transit-routing: "1"
-  natural-earth-source-url: https://data.example.com/0.6.0/natural_earth.mbtiles
-  mbtiles-source-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.mbtiles
+  terrain-source-url: https://data.example.com/0.6.0/terrain.mbtiles
+  landcover-source-url: https://data.example.com/0.6.0/landcover.mbtiles
+  areamap-source-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.mbtiles
   valhalla-artifact-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.valhalla.tar.zst
   placeholder-artifact-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.placeholder.tar.zst
   elasticsearch-artifact-url: https://data.example.com/0.6.0/maps-earth-planet-v1.36/maps-earth-planet-v1.36.elasticsearch.tar.zst

diff --git a/k8s/configs/planet/tileserver-deployment.yaml b/k8s/configs/planet/tileserver-deployment.yaml
@@ -22,20 +22,27 @@ spec:
           env:
             - name: HEADWAY_AREA
               value: area
-            - name: NATURAL_EARTH_ARTIFACT_DEST_PATH
-              value: /data/natural_earth.mbtiles
-            - name: NATURAL_EARTH_ARTIFACT_URL
+            - name: TERRAIN_ARTIFACT_DEST
+              value: /data/terrain.mbtiles
+            - name: TERRAIN_ARTIFACT_SOURCE
               valueFrom:
                 configMapKeyRef:
                   name: deployment-config
-                  key: natural-earth-source-url
-            - name: MBTILES_ARTIFACT_DEST_PATH
+                  key: terrain-source-url
+            - name: LANDCOVER_ARTIFACT_DEST
+              value: /data/landcover.mbtiles
+            - name: LANDCOVER_ARTIFACT_SOURCE
+              valueFrom:
+                configMapKeyRef:
+                  name: deployment-config
+                  key: landcover-source-url
+            - name: AREAMAP_ARTIFACT_DEST
               value: /data/area.mbtiles
-            - name: MBTILES_ARTIFACT_URL
+            - name: AREAMAP_ARTIFACT_SOURCE
               valueFrom:
                 configMapKeyRef:
                   name: deployment-config
-                  key: mbtiles-source-url
+                  key: areamap-source-url
           resources:
             limits:
               memory: 200Mi

diff --git a/k8s/configs/seattle-dev/deployment-config.yaml b/k8s/configs/seattle-dev/deployment-config.yaml
@@ -7,8 +7,9 @@ data:
   public-url: https://seattle.maps.earth
   bbox: "-122.462 47.394 -122.005 47.831"
   enable-transit-routing: "1"
-  natural-earth-source-url: https://data.example.com/dev/natural_earth.mbtiles
-  mbtiles-source-url: https://data.example.com/dev/seattle-latest/Seattle.mbtiles
+  terrain-source-url: https://data.example.com/dev/terrain.mbtiles
+  landcover-source-url: https://data.example.com/dev/landcover.mbtiles
+  areamap-source-url: https://data.example.com/dev/seattle-latest/Seattle.mbtiles
   valhalla-artifact-url: https://data.example.com/dev/seattle-latest/Seattle.valhalla.tar.zst
   placeholder-artifact-url: https://data.example.com/dev/seattle-latest/Seattle.placeholder.tar.zst
   elasticsearch-artifact-url: https://data.example.com/dev/seattle-latest/Seattle.elasticsearch.tar.zst

diff --git a/k8s/configs/seattle-dev/tileserver-deployment.yaml b/k8s/configs/seattle-dev/tileserver-deployment.yaml
@@ -22,20 +22,27 @@ spec:
           env:
             - name: HEADWAY_AREA
               value: area
-            - name: NATURAL_EARTH_ARTIFACT_DEST_PATH
-              value: /data/natural_earth.mbtiles
-            - name: NATURAL_EARTH_ARTIFACT_URL
+            - name: TERRAIN_ARTIFACT_DEST
+              value: /data/terrain.mbtiles
+            - name: TERRAIN_ARTIFACT_SOURCE
               valueFrom:
                 configMapKeyRef:
                   name: deployment-config
-                  key: natural-earth-source-url
-            - name: MBTILES_ARTIFACT_DEST_PATH
+                  key: terrain-source-url
+            - name: LANDCOVER_ARTIFACT_DEST
+              value: /data/landcover.mbtiles
+            - name: LANDCOVER_ARTIFACT_SOURCE
+              valueFrom:
+                configMapKeyRef:
+                  name: deployment-config
+                  key: landcover-source-url
+            - name: AREAMAP_ARTIFACT_DEST
               value: /data/area.mbtiles
-            - name: MBTILES_ARTIFACT_URL
+            - name: AREAMAP_ARTIFACT_SOURCE
               valueFrom:
                 configMapKeyRef:
                   name: deployment-config
-                  key: mbtiles-source-url
+                  key: areamap-source-url
           resources:
             limits:
               memory: 200Mi

diff --git a/services/tileserver/init.sh b/services/tileserver/init.sh
@@ -3,38 +3,30 @@
 set -xe
 set -o pipefail
 
-mkdir -p $(dirname ${MBTILES_ARTIFACT_DEST_PATH})
+function download() {
+    local source_path=$1
+    local dest_path=$2
 
-if [ -f "${MBTILES_ARTIFACT_DEST_PATH}" ]; then
-    echo "Nothing to do, already have ${MBTILES_ARTIFACT_DEST_PATH}"
-elif [ -f "${MBTILES_ARTIFACT_SOURCE_PATH}" ]; then
-    echo "Copying mbtiles artifact."
-    cp "${MBTILES_ARTIFACT_SOURCE_PATH}" "${MBTILES_ARTIFACT_DEST_PATH}"
-elif [ ! -z "${MBTILES_ARTIFACT_URL}" ]; then
-    echo "Downloading mbtiles artifact."
+    mkdir -p "$(dirname "$dest_path")"
 
-    wget --tries=100 --continue -O "${MBTILES_ARTIFACT_DEST_PATH}.download" "${MBTILES_ARTIFACT_URL}"
-    WGET_STATUS=$?
-    echo "wget exit code was: ${WGET_STATUS}"
-    echo "Downloaded mbtiles artifact."
-    mv "${MBTILES_ARTIFACT_DEST_PATH}.download" "${MBTILES_ARTIFACT_DEST_PATH}"
-else
-    echo "No 'area' mbtiles artifact available."
-    exit 1
-fi
+    if [[ -f "$dest_path" ]]; then
+        echo "Already have ${dest_path}."
+    elif [[ $source_path == http* ]]; then
+        echo "Downloading ${source_path}..."
+        wget --tries=100 --continue -O "${dest_path}.download" "$source_path"
+        local wget_status=$?
+        echo "wget exit code was: ${wget_status}"
+        mv "${dest_path}.download" "$dest_path"
+    elif [[ -n "$source_path" ]]; then
+        echo "Copying ${source_path}..."
+        cp "$source_path" "$dest_path"
+    else
+        echo "No source specified for ${dest_path}"
+        exit 1
+    fi
+    echo "done"
+}
 
-mkdir -p $(dirname ${NATURAL_EARTH_ARTIFACT_DEST_PATH})
-
-if [ -f "${NATURAL_EARTH_ARTIFACT_DEST_PATH}" ]; then
-    echo "Nothing to do, already have ${NATURAL_EARTH_ARTIFACT_DEST_PATH}"
-elif [ -f "${NATURAL_EARTH_ARTIFACT_SOURCE_PATH}" ]; then
-    echo "Copying natural earth artifact."
-    cp "${NATURAL_EARTH_ARTIFACT_SOURCE_PATH}" "${NATURAL_EARTH_ARTIFACT_DEST_PATH}"
-elif [ ! -z "${NATURAL_EARTH_ARTIFACT_URL}" ]; then
-    echo "Downloading natural earth artifact."
-    wget --tries=100 --continue -O "${NATURAL_EARTH_ARTIFACT_DEST_PATH}.download" "${NATURAL_EARTH_ARTIFACT_URL}"
-    mv "${NATURAL_EARTH_ARTIFACT_DEST_PATH}.download" "${NATURAL_EARTH_ARTIFACT_DEST_PATH}"
-else
-    echo "No 'natural earth' mbtiles artifact available."
-    exit 1
-fi
+download "$AREAMAP_ARTIFACT_SOURCE" "$AREAMAP_ARTIFACT_DEST"
+download "$TERRAIN_ARTIFACT_SOURCE" "$TERRAIN_ARTIFACT_DEST"
+download "$LANDCOVER_ARTIFACT_SOURCE" "$LANDCOVER_ARTIFACT_DEST"