Merge from develop

images/taginfo/start.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -x
 WORKDIR=/usr/src/app
 DATADIR=/usr/src/app/data
 DATADOWNLOAD=/osm/planet/var
@@ -39,14 +40,14 @@ process_data() {
     db/update.sh $DATADIR
     master/update.sh $DATADIR
     projects/update.sh $DATADIR
-    # # languages/update.sh $DATADIR
-    # # wiki/update.sh $DATADIR
-    # # wikidata/update.sh $DATADIR
+    cp $DATADIR/selection.db $DATADIR/../
+    # languages/update.sh $DATADIR
+    # wiki/update.sh $DATADIR
+    # wikidata/update.sh $DATADIR
     chronology/update.sh $DATADIR
     ./update_all.sh $DATADIR
     mv $DATADIR/*.db $DATADIR/
     mv $DATADIR/*/*.db $DATADIR/
-
     # if BUCKET_NAME is set upload data
     if ! aws s3 ls "s3://$BUCKET_NAME/$ENVIRONMENT" 2>&1 | grep -q 'An error occurred'; then
         aws s3 sync $DATADIR/ s3://$AWS_S3_BUCKET/$ENVIRONMENT/  --exclude "*" --include "*.db"
@@ -64,6 +65,8 @@ compress_files() {
 download_db_files() {
     if ! aws s3 ls "s3://$AWS_S3_BUCKET/$ENVIRONMENT" 2>&1 | grep -q 'An error occurred'; then
         aws s3 sync "s3://$AWS_S3_BUCKET/$ENVIRONMENT/" "$DATADIR/"
+        mv $DATADIR/*.db $DATADIR/
+        mv $DATADIR/*/*.db $DATADIR/
         compress_files
     fi
 }

images/tiler-imposm/Dockerfile

@@ -41,6 +41,8 @@ ENV PATH $PATH:/usr/local/go/bin
 WORKDIR /go
 ENV GOPATH /go
 RUN go get github.com/omniscale/imposm3
+WORKDIR $GOPATH/src/github.com/omniscale/imposm3
+RUN git checkout v0.11.1
 RUN go install github.com/omniscale/imposm3/cmd/imposm
 ENV PATH $PATH:$GOPATH/bin
 

images/tiler-imposm/start.sh

@@ -1,16 +1,15 @@
 #!/bin/bash
-set -e
-
-STATEFILE="state.txt"
-PBFFILE="osm.pbf"
-LIMITFILE="limitFile.geojson"
+set -ex
 
 # directories to keep the imposm's cache for updating the db
 WORKDIR=/mnt/data
 CACHE_DIR=$WORKDIR/cachedir
 DIFF_DIR=$WORKDIR/diff
 IMPOSM3_EXPIRE_DIR=$WORKDIR/imposm3_expire_dir
 
+PBFFILE="${WORKDIR}/osm.pbf"
+STATEFILE="state.txt"
+LIMITFILE="limitFile.geojson"
 # # Setting directory
 # settingDir=/osm
 # Folder to store the imposm expider files in s3 or gs
@@ -33,18 +32,9 @@ mkdir -p "$CACHE_DIR" "$DIFF_DIR" "$IMPOSM3_EXPIRE_DIR"
 function getData() {
     ### Get the PBF file from the cloud provider or public URL
     if [ "$TILER_IMPORT_FROM" == "osm" ]; then
-        wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE"
-    elif [ "$TILER_IMPORT_FROM" == "osmseed" ]; then
-        if [ "$CLOUDPROVIDER" == "aws" ]; then
-            # Get the state.txt file from S3
-            aws s3 cp "$AWS_S3_BUCKET/planet/full-history/$STATEFILE" .
-            PBFCloudPath=$(tail -n +1 "$STATEFILE")
-            aws s3 cp "$PBFCloudPath" "$PBFFILE"
-        elif [ "$CLOUDPROVIDER" == "gcp" ]; then
-            # Get the state.txt file from GS
-            gsutil cp "$GCP_STORAGE_BUCKET/planet/full-history/$STATEFILE" .
-            PBFCloudPath=$(tail -n +1 "$STATEFILE")
-            gsutil cp "$PBFCloudPath" "$PBFFILE"
+        if [ ! -f "$PBFFILE" ]; then
+          echo "$PBFFILE does not exist, downloading..."
+          wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE"
         fi
     fi
 }
@@ -123,26 +113,30 @@ function importData() {
     echo "Execute the missing functions"
     psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_helpers.sql
 
-    echo "Import Natural Earth..."
-    ./scripts/natural_earth.sh
-
-    echo "Import OSM Land..."
-    ./scripts/osm_land.sh
+    if [ "$IMPORT_NATURAL_EARTH" = "true" ]; then
+        echo "Importing Natural Earth..."
+        ./scripts/natural_earth.sh
+    fi
 
+    if [ "$IMPORT_OSM_LAND" = "true" ]; then
+        echo "Import OSM Land..."
+        ./scripts/osm_land.sh
+    fi
+
     echo "Import PBF file..."
     if [ -z "$TILER_IMPORT_LIMIT" ]; then
         imposm import \
             -config $WORKDIR/config.json \
             -read $PBFFILE \
             -write \
-            -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR
+            -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR
     else
         wget $TILER_IMPORT_LIMIT -O $WORKDIR/$LIMITFILE
         imposm import \
             -config $WORKDIR/config.json \
             -read $PBFFILE \
             -write \
-            -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR \
+            -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR \
             -limitto $WORKDIR/$LIMITFILE
     fi
 

osm-seed/templates/db/db-statefulset.yaml

@@ -43,10 +43,14 @@ spec:
               value: {{ default "postgres" .Values.db.env.POSTGRES_USER | quote }}
             - name: PGUSER
               value: {{ default "postgres" .Values.db.env.POSTGRES_USER | quote }}
+            - name: ENVIRONMENT
+              value: {{ .Values.environment }}
             - name: PGDATA
               value: {{ .Values.db.persistenceDisk.mountPath }}
             - name: POD_IP
               valueFrom: { fieldRef: { fieldPath: status.podIP } }
+            - name: LOG_STATEMENT
+              value: {{ default "none" .Values.db.env.LOG_STATEMENT | quote }}
           livenessProbe:
             exec:
               command:

osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml

@@ -1,5 +1,4 @@
 {{- if .Values.tilerImposm.enabled -}}
-{{- if .Values.tilerImposm.persistenceDisk.enabled -}}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -66,6 +65,10 @@ spec:
             value: {{ .Values.tilerImposm.env.OVERWRITE_STATE | quote }}
           - name: UPLOAD_EXPIRED_FILES
             value: {{ .Values.tilerImposm.env.UPLOAD_EXPIRED_FILES | default "true" | quote }}
+          - name: IMPORT_NATURAL_EARTH
+            value: {{ .Values.tilerImposm.env.IMPORT_NATURAL_EARTH | default "true" | quote }}
+          - name: IMPORT_OSM_LAND
+            value: {{ .Values.tilerImposm.env.IMPORT_OSM_LAND | default "true" | quote }}
           - name: CLOUDPROVIDER
             value: {{ .Values.cloudProvider }}
           # In case cloudProvider=aws
@@ -95,4 +98,3 @@ spec:
         {{ .Values.tilerImposm.nodeSelector.label_key }} : {{ .Values.tilerImposm.nodeSelector.label_value }}
       {{- end }}
 {{- end }}
-{{- end }}

osm-seed/templates/web/web-deployment.yaml

@@ -125,6 +125,12 @@ spec:
               value: {{ .Values.web.env.NEW_RELIC_APP_NAME | default ""}}
             - name: ORGANIZATION_NAME
               value: {{ .Values.web.env.ORGANIZATION_NAME | default "OpenStreetMap"}}
+            - name: WEBSITE_STATUS
+              value: {{ .Values.web.env.WEBSITE_STATUS | default "online"}}
+            - name: API_TIMEOUT
+              value: {{ .Values.web.env.API_TIMEOUT | default "30" | quote }}
+            - name: WEB_TIMEOUT
+              value: {{ .Values.web.env.WEB_TIMEOUT | default "30" | quote }}
           volumeMounts:
             - mountPath: /dev/shm
               name: shared-memory

osm-seed/templates/web/web-ingress.yaml

@@ -6,7 +6,7 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: nginx
     cert-manager.io/cluster-issuer: letsencrypt-prod-issuer
-    nginx.ingress.kubernetes.io/proxy-body-size: 20m
+    nginx.ingress.kubernetes.io/proxy-body-size: 100m
 spec:
   ingressClassName: nginx
   tls:

osm-seed/values.yaml

@@ -75,6 +75,7 @@ db:
     POSTGRES_USER: postgres
     POSTGRES_PASSWORD: "1234"
     ENVIRONMENT: "production"
+    LOG_STATEMENT: "all"
   persistenceDisk:
     enabled: false
     mountPath: /var/lib/postgresql/data
@@ -126,6 +127,9 @@ web:
     OAUTH_CLIENT_ID: "abcd" # Create client ID and key as OAuth2 Application
     OAUTH_KEY: "xyz"
     ORGANIZATION_NAME: "OSMSeed"
+    WEBSITE_STATUS: "online"
+    API_TIMEOUT: 30
+    WEB_TIMEOUT: 30
   resources:
     enabled: false
     requests:
@@ -378,6 +382,8 @@ tilerImposm:
     TILER_IMPORT_PBF_URL: http://download.geofabrik.de/europe/monaco-latest.osm.pbf
     TILER_IMPORT_LIMIT: https://gist.githubusercontent.com/Rub21/96bdcac5eb11f0b36ba8d0352ac537f4/raw/2606f2e207d4a0d895897a83efa1efacefd36eb4/monaco.geojson
     UPLOAD_EXPIRED_FILES: true
+    IMPORT_NATURAL_EARTH: true
+    IMPORT_OSM_LAND: true
   persistenceDisk:
     enabled: false
     accessMode: ReadWriteOnce