diff --git a/docker-compose.yml b/docker-compose.yml index 8ab814d..f1a2748 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,9 @@ services: volumes: - ${APP_CODE_PATH_HOST}:${APP_CODE_PATH_CONTAINER} - ${DATA_PATH_HOST}/pbf:${APP_CODE_PATH_CONTAINER}/pbf + - ./workspace/mapping.yml:/etc/imposm/mapping.yml + - ./workspace/osm-initial-import.sh:/usr/bin/osm-initial-import + extra_hosts: - "dockerhost:${DOCKER_HOST_IP}" tty: true @@ -49,11 +52,11 @@ services: - postgres-postgis - cassandra - redis - - tegola volumes: - ${APP_CODE_PATH_HOST}/kartotherian:/home/kartotherian - ${APP_CODE_PATH_HOST}/dependencies:/srv/dependencies - - ${APP_CODE_PATH_HOST}/kartosm/expiretiles:/srv/expiretiles + - ${DATA_PATH_HOST}/imposm:/srv/imposm + - ./kartotherian/sources.docker.yaml:/etc/opt/sources.docker.yaml tty: true networks: - frontend @@ -63,7 +66,7 @@ services: postgres-postgis: image: postgis/postgis:11-3.1 volumes: - - ${DATA_PATH_HOST}/imposm:/var/lib/postgresql/data + - ${DATA_PATH_HOST}/postgres:/var/lib/postgresql/data ports: - "${PGPORT}:5432" environment: @@ -113,9 +116,17 @@ services: ### Tegola ################################## tegola: - build: https://github.com/go-spatial/tegola.git#v0.14.x + build: ./tegola +# build: https://github.com/go-spatial/tegola.git#v0.14.x + environment: + - TEGOLA_BROKER_URL=redis://redis:6379 + - TEGOLA_QUEUE_NAME=pregen + - TEGOLA_PATH=/opt/tegola + - TEGOLA_CONFIG_PATH=/etc/tegola.toml volumes: - ./tegola/tegola.toml:/etc/tegola.toml + - ./tegola/pregenerate-maps-tile.sh:/etc/pregenerate-maps-tile.sh + - ${DATA_PATH_HOST}/tegola/cache:/etc/cache ports: - "${TEGOLA_PORT}:8080" networks: diff --git a/kartotherian/Dockerfile b/kartotherian/Dockerfile index b5598bb..9e12d0f 100644 --- a/kartotherian/Dockerfile +++ b/kartotherian/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update && apt-get install -y nodejs \ ENV NVM_DIR $HOME/.nvm # Avoid using cached layers from this point to allow easy changing of nvm version install -ARG CACHEBUST=1 +ARG CACHEBUST=1 RUN wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.6/install.sh | bash \ && . $NVM_DIR/nvm.sh && nvm install 10.15.2 @@ -47,7 +47,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends yarn COPY config.tilerator.docker.template.yaml /etc/opt/config.tilerator.docker.template.yaml COPY config.kartotherian.docker.template.yaml /etc/opt/config.kartotherian.docker.template.yaml -COPY sources.docker.yaml /etc/opt/sources.docker.yaml COPY generate_config.sh /usr/local/bin/generate_config RUN chmod +x /usr/local/bin/generate_config diff --git a/kartotherian/config.kartotherian.docker.template.yaml b/kartotherian/config.kartotherian.docker.template.yaml index bb0cafc..a5d736d 100644 --- a/kartotherian/config.kartotherian.docker.template.yaml +++ b/kartotherian/config.kartotherian.docker.template.yaml @@ -87,6 +87,7 @@ services: - "@wikimedia/kartotherian-overzoom" - "@wikimedia/kartotherian-postgres" - "@wikimedia/kartotherian-substantial" + - "@wikimedia/kartotherian-tilelive-switch" - "@kartotherian/tilelive-tmsource" - "@kartotherian/tilelive-vector" - "@mapbox/tilejson" diff --git a/kartotherian/sources.docker.yaml b/kartotherian/sources.docker.yaml index f48e102..92d7141 100644 --- a/kartotherian/sources.docker.yaml +++ b/kartotherian/sources.docker.yaml @@ -15,11 +15,6 @@ genraw: port: { var: osmdb-port } user: {var: osmdb-user} password: {var: osmdb-pswd} -# expand name_ attributes into individual tags -gen: - uri: json2tags:// - params: - source: {ref: genraw} # Tegola backend http-tegola: @@ -38,6 +33,49 @@ http-tegola: - id: poi_label - id: road_label +# Tegola beta-cluster backend +beta-cluster-http-tegola: + uri: https://tegola.wmflabs.org/maps/osm/{z}/{x}/{y}.pbf + setInfo: + vector_layers: + - id: landuse + - id: waterway + - id: water + - id: aeroway + - id: building + - id: road + - id: admin + - id: country_label + - id: place_label + - id: poi_label + - id: road_label + +# expand name_ attributes into individual tags +gen-tegola: + uri: json2tags:// + params: +# source: {ref: beta-cluster-http-tegola} + source: {ref: http-tegola} + +# Mirror requests to a secondary source +switch: + uri: mirror:// + params: + source: {ref: genraw} + secondarySource: { ref: beta-cluster-http-tegola } + enableMirror: true + loadBalancer: + bucketSize: 1000 # milliseconds + totalBuckets: 10 # number of buckets in the histogram + threshold: 0.5 # % of load to the mirrored source + +# expand name_ attributes into individual tags +gen: + uri: json2tags:// + params: + source: {ref: switch} + + # Cassandra storage v4: public: true @@ -82,119 +120,15 @@ genview: Pragma: no-cache Expires: 0 -# Filter out any small tiles without significant data on or after zoom 10 -substgen: - public: true - formats: [pbf] - uri: substantial:// - params: - source: {ref: gen} - minzoom: 10 - minsize: 500 - layers: [water, landuse] - -# If tile has been filtered out by substgen, use overzoom to recreate it -substgenoz: - public: true - formats: [pbf] - uri: overzoom:// - params: - source: {ref: substgen} - -# View tiles from generator, without the insignificant ones (will show blank squares) -substgenview: - public: true - formats: [png,json,headers,svg,jpeg] - scales: [1.3, 1.5, 2, 2.6, 3] - static: true - maxheight: 2048 - maxwidth: 2048 - uri: tmstyle:// - yaml: - npm: ["@kartotherian/osm-bright-style", "project.yml"] - yamlSetParams: - source: {ref: substgen} - -# View tiles from generator, and use overzoom for any filtered out ones -substgenozview: - public: true - formats: [png,json,headers,svg,jpeg] - scales: [1.3, 1.5, 2, 2.6, 3] - static: true - maxheight: 2048 - maxwidth: 2048 - uri: tmstyle:// - yaml: - npm: ["@kartotherian/osm-bright-style", "project.yml"] - yamlSetParams: - source: {ref: substgenoz} - -# Filter out any tiles stored in the v4 that are not significant -substv4: - public: true - formats: [pbf] - uri: substantial:// - params: - source: {ref: v4} - minzoom: 10 - minsize: 500 - layers: [water, landuse] - -# Use overzoom on v4 tiles for any filtered out insignificant tiles -substv4oz: - public: true - formats: [pbf] - uri: overzoom:// - params: - source: {ref: substv4} - -# Show v4 tiles with insignificant ones showing as blank squares -substv4view: - public: true - formats: [png,json,headers,svg,jpeg] - scales: [1.3, 1.5, 2, 2.6, 3] - static: true - maxheight: 2048 - maxwidth: 2048 - uri: tmstyle:// - yaml: - npm: ["@kartotherian/osm-bright-style", "project.yml"] - yamlSetParams: - source: {ref: substv4} - -# Show v4 tiles and use overzoom for those that have been filtered out -substv4ozview: - public: true - formats: [png,json,headers,svg,jpeg] - scales: [1.3, 1.5, 2, 2.6, 3] - static: true - maxheight: 2048 - maxwidth: 2048 - uri: tmstyle:// - yaml: - npm: ["@kartotherian/osm-bright-style", "project.yml"] - yamlSetParams: - source: {ref: substv4oz} - # Default OSM PBF source osm-pbf: public: true formats: [pbf] uri: overzoom:// params: - source: {ref: v4} - maxzoom: 19 - overrideInfo: - attribution: 'Map data © OpenStreetMap contributors' - tiles: ["http://localhost:6533/osm-pbf/{z}/{x}/{y}.pbf"] - -# Default OSM PBF source -osm-pbf-tegola: - public: true - formats: [pbf] - uri: overzoom:// - params: - source: {ref: http-tegola} + source: {ref: gen-tegola} +# source: {ref: beta-cluster-http-tegola} +# source: { ref: v4 } maxzoom: 19 overrideInfo: attribution: 'Map data © OpenStreetMap contributors' @@ -208,38 +142,7 @@ babel: combineName: false defaultLanguage: local -babel-tegola: - uri: babel:// - params: - source: {ref: osm-pbf-tegola} - tag: name - combineName: false - defaultLanguage: local - -# OSM map with international labeling - tegola backed -osm-intl-tegola: - public: true - formats: [png,json,headers,svg,jpeg] - scales: [1.3, 1.5, 2, 2.6, 3] - maxzoom: 19 - static: true - maxheight: 2048 - maxwidth: 2048 - uri: tmstyle:// - yaml: - npm: ["@kartotherian/osm-bright-style", "project.yml"] - yamlSetParams: - source: {ref: babel-tegola} - overrideInfo: - attribution: 'Map data © OpenStreetMap contributors' - tiles: ["http://localhost:6533/osm-intl-tegola/{z}/{x}/{y}.png"] - defaultHeaders: - Cache-Control: 'no-cache, no-store, must-revalidate' - Pragma: no-cache - Expires: 0 - - -# OSM map with international labeling - will be used as default +# OSM map with international labeling osm-intl: public: true formats: [png,json,headers,svg,jpeg] diff --git a/makefile b/makefile index 443f3ce..a5d4daa 100644 --- a/makefile +++ b/makefile @@ -11,9 +11,6 @@ osm: generate_config: docker-compose exec kartotherian generate_config -generate_config_osm2pgsql: - docker-compose exec kartotherian-osm2pgsql generate_config - # Cassandra Keyspace setup https://wikitech.wikimedia.org/wiki/Maps/Keyspace_Setup keyspace_setup: docker-compose exec kartotherian bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/tilerator/scripts/tileshell.js --config /etc/opt/config.tilerator.docker.yaml --source /etc/opt/sources.docker.yaml" @@ -24,12 +21,6 @@ run_tilerator: run_kartotherian: docker-compose exec kartotherian bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/kartotherian/server.js -c /etc/opt/config.kartotherian.docker.yaml" -run_tilerator_osm2pgsql: - docker-compose exec kartotherian-osm2pgsql bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/tilerator/server.js -c /etc/opt/config.tilerator.docker.yaml" - -run_kartotherian_osm2pgsql: - docker-compose exec kartotherian-osm2pgsql bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/kartotherian/server.js -c /etc/opt/config.kartotherian.docker.yaml" - npm_test: docker-compose exec kartotherian bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && npm test" @@ -48,11 +39,16 @@ clean: docker-compose exec kartotherian bash -c "./clean_node_modules.sh" imposm_run: - docker-compose exec workspace bash -c "imposm run -config /srv/kartosm/config.json" -expiretiles-zoom 15 + docker-compose exec workspace bash -c "imposm run -config /etc/imposm/config.json" -expiretiles-zoom 15 notify_tilerator: docker-compose exec kartotherian bash -c ". /.nvm/nvm.sh && nvm use 10.15.2 && node /home/kartotherian/packages/tilerator/scripts/tileshell.js --config /etc/opt/config.tilerator.docker.yaml -j.fromZoom 10 -j.beforeZoom 16 -j.generatorId gen -j.storageId v4 -j.deleteEmpty -j.expdirpath /srv/expiretiles -j.expmask '(expire\.list\.*)|(\.tiles)' -j.statefile /home/kartotherian/expire.state" +pregen_dequeue: + docker-compose exec tegola /etc/pregenerate-maps-tile.sh +pregen_enqueue: + docker-compose exec tegola poppy --broker-url redis://redis:6379 --queue-name pregen enqueue --message-input tile 1/1/1 + install: # TODO # Check if kartotherian is installed and clone if it isn't diff --git a/tegola/Dockerfile b/tegola/Dockerfile new file mode 100644 index 0000000..3864263 --- /dev/null +++ b/tegola/Dockerfile @@ -0,0 +1,38 @@ +# Based on upstream Dockerfile https://github.com/go-spatial/tegola/blob/v0.14.x/Dockerfile +FROM golang:1.16.2-alpine3.12 AS build + +ARG VERSION="Version Not Set" +ENV VERSION="${VERSION}" + +# Only needed for CGO support at time of build, results in no noticable change in binary size +# incurs approximately 1:30 extra build time (1:54 vs 0:27) to install packages. Doesn't impact +# development as these layers are drawn from cache after the first build. +RUN apk update \ + && apk add musl-dev=1.1.24-r10 \ + && apk add gcc=9.3.0-r2 \ + && apk add git + +# Set up source for compilation +RUN mkdir -p /go/src/github.com/go-spatial/tegola +RUN git clone "https://gerrit.wikimedia.org/r/operations/software/tegola" /go/src/github.com/go-spatial/tegola + +# Build binary +RUN cd /go/src/github.com/go-spatial/tegola/cmd/tegola \ + && go build -v -ldflags "-w -X 'github.com/go-spatial/tegola/cmd/tegola/cmd.Version=${VERSION}'" -gcflags "-N -l" -o /opt/tegola \ + && chmod a+x /opt/tegola + +# Create minimal deployment image, just alpine & the binary +FROM python:3-alpine3.12 + +RUN apk update \ + && apk add ca-certificates \ + && apk add git \ + && rm -rf /var/cache/apk/* + +RUN pip install "git+https://github.com/wikimedia/poppy-cli@dequeue-until-empty" \ + redis \ + kafka-python + +COPY --from=build /opt/tegola /opt/ +WORKDIR /opt +ENTRYPOINT ["/opt/tegola"] diff --git a/tegola/pregenerate-maps-tile.sh b/tegola/pregenerate-maps-tile.sh new file mode 100755 index 0000000..f3041f4 --- /dev/null +++ b/tegola/pregenerate-maps-tile.sh @@ -0,0 +1,44 @@ +#!/bin/sh +if [ -z "$TEGOLA_BROKER_URL" ] +then + echo "TEGOLA_BROKER_URL env var is not set" + exit 1 +fi +if [ -z "$TEGOLA_QUEUE_NAME" ] +then + echo "TEGOLA_QUEUE_NAME env var is not set" + exit 1 +fi +if [ -z "$TEGOLA_PATH" ] +then + echo "TEGOLA_PATH env var is not set" + exit 1 +fi +if [ -z "$TEGOLA_CONFIG_PATH" ] +then + echo "TEGOLA_CONFIG_PATH env var is not set" + exit 1 +fi +TMP_DIR=$(mktemp -d /tmp/tegola-XXXXXXXXXX) +TILELIST_PATH=${TEGOLA_TILELIST_DIR:-$TMP_DIR}/tilelist.txt +BATCH_SIZE=${TEGOLA_PREGENERATION_BATCH_SIZE:-1000} +DEQUEUE_TIMEOUT=${TEGOLA_PREGENERATION_DEQUEUE_TIMEOUT:-60} +set -x +while true; +do + # Dequeue a batch of messages from the queue and store them in tilelist + poppy --broker-url "$TEGOLA_BROKER_URL" \ + --queue-name "$TEGOLA_QUEUE_NAME" \ + dequeue --batch "$BATCH_SIZE" --exit-on-empty True --dequeue-raise-on-empty True --blocking-dequeue-timeout "$DEQUEUE_TIMEOUT" > "$TILELIST_PATH" + status=$? + # Pregenerate tiles that exist in tilelist + $TEGOLA_PATH --config "$TEGOLA_CONFIG_PATH" cache seed tile-list "$TILELIST_PATH" + if [ $status -eq 100 ] # Queue is empty + then + exit 0 + elif [ $status -gt 0 ] # Something went wrong + then + exit 1 + fi +done +set +x diff --git a/tegola/tegola.toml b/tegola/tegola.toml index 354e3c1..2fea6e8 100644 --- a/tegola/tegola.toml +++ b/tegola/tegola.toml @@ -1,8 +1,15 @@ +[cache] +type = "file" +basepath = "/etc/cache" + +[observer] +type = "prometheus" + [[providers]] name = "osm" -type = "postgis" +type = "mvt_postgis" host = "postgres-postgis" -max_connections = 10 +max_connections = 50 port = 5432 database = "gis" user = "default" @@ -10,540 +17,94 @@ password = "secret" [[providers.layers]] name = "landuse" - geometry_fieldname = "way" - geometry_type = "Polygon" - fields = [ "class", "osm_id", "way_area", "z_order" ] - sql = """ - SELECT - osm_id, - ST_AsBinary(way) AS way, - CASE - WHEN \"natural\" = 'wood' OR landuse IN ('wood', 'forest') THEN 'wood' - WHEN leisure IN ('national_reserve', 'nature_reserve', 'golf_course') OR boundary = 'national_park' THEN 'park' - WHEN landuse IN ('cemetery', 'industrial') THEN landuse - WHEN aeroway IS NOT NULL AND aeroway <> '' THEN 'industrial' - WHEN landuse = 'village_green' OR leisure IN ('park', 'playground') THEN 'park' - WHEN amenity IN ('school', 'university') THEN 'school' - WHEN amenity = 'hospital' THEN 'hospital' - ELSE bail_out('Unexpected landuse row with osm_id=%s', osm_id::TEXT) - END AS class, - z_order, - way_area - FROM planet_osm_polygon - WHERE - ( - ( - ( - \"natural\" = 'wood' OR landuse IN ('wood', 'forest') - OR leisure IN ('national_reserve', 'nature_reserve', 'golf_course') - OR boundary = 'national_park' - ) - AND z(!SCALE_DENOMINATOR!) >= 7 - ) OR ( - ( - landuse IN ('cemetery', 'industrial', 'village_green') - OR (aeroway IS NOT NULL AND aeroway <> '') - OR leisure IN ('park', 'playground') - OR amenity IN ('school', 'university') - ) - AND z(!SCALE_DENOMINATOR!) >= 10 - ) OR ( - amenity = 'hospital' - AND z(!SCALE_DENOMINATOR!) >= 12 - ) - ) - AND way && !BBOX! - ORDER BY z_order, way_area DESC - """ + geometry_type = "polygon" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT class, osm_id, way_area, z_order, ST_AsMVTGeom(geometry, !BBOX!) AS geom FROM layer_landuse(!BBOX!, !ZOOM!)" [[providers.layers]] name = "waterway" - geometry_fieldname = "way" - geometry_type = "LineString" - fields = [ "class", "osm_id" ] - sql = """ - SELECT osm_id, ST_AsBinary(way) AS way, waterway AS class - FROM planet_osm_line - WHERE - ( - ( - waterway IN ('river', 'canal') - AND z(!SCALE_DENOMINATOR!) >= 8 - ) - OR - ( - waterway IN ('stream', 'stream_intermittent') - AND z(!SCALE_DENOMINATOR!) >= 13 - ) - ) - AND way && !BBOX! - """ + geometry_type = "linestring" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT class, osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom FROM layer_waterway(!BBOX!, !ZOOM!)" [[providers.layers]] name = "water" - geometry_type = "Polygon" - geometry_fieldname = "way" - fields = [ "osm_id" ] - sql = """ - SELECT osm_id, ST_AsBinary(way) AS way - FROM planet_osm_polygon - WHERE - ( - \"natural\" = 'water' - OR (waterway IS NOT NULL AND waterway <> '') - OR landuse = 'reservoir' - OR landuse = 'pond' - ) - AND - ( - z(!SCALE_DENOMINATOR!) >= 14 - OR way_area >= 5000000000 / 2.3^z(!SCALE_DENOMINATOR!) - ) - AND way && !BBOX! - UNION ALL - SELECT 0 AS osm_id, ST_AsBinary(way) - FROM water_polygons - WHERE - way && !BBOX! - """ + geometry_type = "polygon" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom FROM layer_water(!BBOX!, !ZOOM!)" [[providers.layers]] name = "aeroway" - geometry_type = "GeometryCollection" - geometry_fieldname = "way" - fields = [ "osm_id", "type" ] - sql = """ - SELECT osm_id, ST_AsBinary(way) AS way, aeroway AS type - FROM planet_osm_polygon - WHERE - (aeroway IS NOT NULL AND aeroway <> '') - AND aeroway IN ('apron', 'helipad', 'runway', 'taxiway') - AND z(!SCALE_DENOMINATOR!) >= 12 - AND way && !BBOX! - UNION ALL - SELECT osm_id, ST_AsBinary(way), aeroway AS type - FROM planet_osm_line - WHERE - (aeroway IS NOT NULL AND aeroway <> '') - AND z(!SCALE_DENOMINATOR!) >= 12 - AND way && !BBOX! - """ + geometry_type = "polygon" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom, type FROM layer_aeroway(!BBOX!, !ZOOM!)" [[providers.layers]] name = "road" - geometry_type = "LineString" - geometry_fieldname = "way" - fields = [ "class", "is", "osm_id" ] - sql = """ - SELECT osm_id, ST_AsBinary(way) AS way, class, \"is\" FROM ( - SELECT - osm_id, - way, - CASE - WHEN highway IN ('motorway', 'motorway_link', 'driveway') THEN highway - WHEN highway IN ('primary', 'primary_link', 'trunk', 'trunk_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') THEN 'main' - WHEN highway IN ('residential', 'unclassified', 'living_street') THEN 'street' - WHEN highway IN ('pedestrian', 'construction') OR access = 'private' THEN 'street_limited' - WHEN railway IN ('rail', 'monorail', 'narrow_gauge', 'subway', 'tram') THEN 'major_rail' - WHEN highway IN ('service', 'track') THEN 'service' - WHEN highway IN ('path', 'cycleway', 'ski', 'steps', 'bridleway', 'footway') THEN 'path' - WHEN railway IN ('funicular', 'light_rail', 'preserved') THEN 'minor_rail' - ELSE bail_out('Unexpected road row with osm_id=%s', osm_id::TEXT) - END AS class, - z_order, - CASE - WHEN bridge IS NOT NULL AND bridge <> '' AND bridge <> 'no' AND bridge <> '0' THEN 'bridge' - WHEN tunnel IS NOT NULL AND tunnel <> '' AND tunnel <> 'no' AND tunnel <> '0' THEN 'tunnel' - ELSE 'road' - END AS \"is\" - FROM planet_osm_line - WHERE - ( - ( - highway IN ('motorway', -- 'motorway' - 'primary', 'primary_link', 'trunk', 'trunk_link' -- 'main' - ) - AND z(!SCALE_DENOMINATOR!) >= 6 - ) - OR - ( -- 'main' - highway IN ('secondary', 'secondary_link') - AND z(!SCALE_DENOMINATOR!) >= 9 - ) - OR - ( -- 'main' - highway IN ('tertiary', 'tertiary_link') - AND z(!SCALE_DENOMINATOR!) >= 12 - ) - OR - ( -- 'street' - highway IN ('residential', 'unclassified', 'living_street') - AND z(!SCALE_DENOMINATOR!) >= 12 - ) - OR - ( -- 'street_limited' - (highway IN ('pedestrian', 'construction') OR access = 'private') - AND z(!SCALE_DENOMINATOR!) >= 12 - ) - OR - ( -- 'major_rail' - railway IN ('rail', 'monorail', 'narrow_gauge', 'subway', 'tram') - AND z(!SCALE_DENOMINATOR!) >= 12 - ) - OR - ( -- 'motorway_link' - highway IN ('motorway_link') - AND z(!SCALE_DENOMINATOR!) >= 13 - ) - OR - ( -- 'service' - highway IN ('service', 'track') - AND z(!SCALE_DENOMINATOR!) >= 14 - ) - OR - ( -- 'driveway' - highway IN ('driveway') - AND z(!SCALE_DENOMINATOR!) >= 14 - ) - OR - ( -- 'path' - highway IN ('path', 'cycleway', 'ski', 'steps', 'bridleway', 'footway') - AND z(!SCALE_DENOMINATOR!) >= 14 - ) - OR - ( -- 'minor_rail' - railway IN ('funicular', 'light_rail', 'preserved') - AND z(!SCALE_DENOMINATOR!) >= 14 - ) - ) - AND way && !BBOX! - ) data JOIN ( - VALUES - ('motorway', 1000), - ('main', 900), - ('street', 800), - ('motorway_link', 700), - ('street_limited', 600), - ('driveway', 500), - ('major_rail', 400), - ('service', 300), - ('minor_rail', 200), - ('path', 100) - ) AS ordertable(feature, prio) ON class=feature - ORDER BY z_order + prio + - CASE \"is\" - WHEN 'tunnel' THEN -100000 - WHEN 'road' THEN 0 - WHEN 'bridge' THEN 100000 - ELSE bail_out('Unexpected row with is=%s, osm_id=%s', \"is\", osm_id::TEXT)::INT - END - """ + geometry_type = "linestring" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom, class, z_order, \"is\" FROM layer_transportation(!BBOX!, !ZOOM!)" [[providers.layers]] name = "admin" - geometry_type = "GeometryCollection" - geometry_fieldname = "way" - fields = [ "admin_level", "disputed", "maritime", "osm_id" ] - sql = """ - SELECT - osm_id, ST_AsBinary(way) AS way, - admin_level::SMALLINT, - maritime, - CASE - WHEN - tags->'disputed' = 'yes' - OR tags->'dispute' = 'yes' - OR (tags->'disputed_by') IS NOT NULL - OR tags->'status' = 'partially_recognized_state' - THEN 1 - ELSE 0 - END AS disputed - FROM admin - WHERE - maritime <> TRUE - AND ( - ( admin_level = '2' AND z(!SCALE_DENOMINATOR!) >= 2 ) - OR ( admin_level = '4' AND z(!SCALE_DENOMINATOR!) >= 3 ) - ) - AND COALESCE(tags->'left:country', '') <> 'Demarcation Zone' - AND COALESCE(tags->'right:country', '') <> 'Demarcation Zone' - AND way && !BBOX! - """ + geometry_type = "linestring" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom, admin_level, maritime, disputed FROM layer_admin(!BBOX!, !ZOOM!)" [[providers.layers]] name = "country_label" - geometry_type = "Point" - geometry_fieldname = "way" - fields = [ "code", "name", "osm_id", "scalerank" ] - sql = """ - SELECT osm_id, ST_AsBinary(way) AS way, name, (hstore_to_json(extract_names(tags)))::text name_, CASE - WHEN to_int(population) >= 250000000 THEN 1 - WHEN to_int(population) BETWEEN 100000000 AND 250000000 THEN 2 - WHEN to_int(population) BETWEEN 50000000 AND 100000000 THEN 3 - WHEN to_int(population) BETWEEN 25000000 AND 50000000 THEN 4 - WHEN to_int(population) BETWEEN 10000000 AND 25000000 THEN 5 - WHEN to_int(population) < 10000000 THEN 6 - END scalerank, - COALESCE(tags->'ISO3166-1', tags->'country_code_iso3166_1_alpha_2') code - FROM planet_osm_point - WHERE - place = 'country' - AND z(!SCALE_DENOMINATOR!) BETWEEN 3 AND 10 - AND way && !BBOX! - ORDER BY to_int(population) DESC NULLS LAST - """ + geometry_type = "point" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom, name, name_, scalerank, code FROM layer_country_labels(!BBOX!, !ZOOM!)" [[providers.layers]] name = "poi_label" - geometry_type = "GeometryCollection" - geometry_fieldname = "way" - fields = [ "localrank", "maki", "name", "osm_id", "scalerank" ] - sql = """ - SELECT osm_id, ST_AsBinary(way1) AS way, name, rank AS scalerank, localrank, maki FROM - ( - SELECT - osm_id, - name, - (hstore_to_json(extract_names(tags)))::text name_, - CASE - WHEN railway='station' THEN 'rail' - WHEN (tags->'subway') IS NOT NULL THEN 'rail-metro' - WHEN highway='bus_stop' THEN 'bus' - WHEN railway='tram_stop' THEN 'rail-light' - WHEN amenity='ferry_terminal' THEN 'ferry' - ELSE bail_out('Cannot classify poi_label, osm_id=%s', osm_id::TEXT) - END AS maki, - 1 AS localrank, - way AS way1 - FROM planet_osm_point - WHERE - z(!SCALE_DENOMINATOR!) >= 14 - AND - ( - (public_transport='stop_position' AND (tags->'subway') IS NOT NULL) - OR railway IN ('station', 'tram_stop') - OR highway='bus_stop' - OR amenity='ferry_terminal' - ) - AND way && !BBOX! - UNION ALL - SELECT - osm_id, - name, - (hstore_to_json(extract_names(tags)))::text name_, - CASE - WHEN railway='station' THEN 'rail' - WHEN (tags->'subway') IS NOT NULL THEN 'rail-metro' - WHEN highway='bus_stop' THEN 'bus' - WHEN railway='tram_stop' THEN 'rail-light' - WHEN amenity='ferry_terminal' THEN 'ferry' - ELSE bail_out('Cannot classify poi_label, osm_id=%s', osm_id::TEXT) - END AS maki, - 1 AS localrank, - ST_Centroid(way) AS way1 - FROM planet_osm_polygon - WHERE - z(!scale_denominator!) >= 14 - AND - ( - (public_transport='stop_position' AND (tags->'subway') IS NOT NULL) - OR railway IN ('station', 'tram_stop') - OR highway='bus_stop' - OR amenity='ferry_terminal' - ) - AND way && !BBOX! - ) data JOIN ( - VALUES - ('rail', 1), - ('rail-metro', 1), - ('rail-light', 1), - ('ferry', 1), - ('bus', 3) - ) AS ranks(class, rank) ON class=maki - ORDER BY scalerank, maki DESC, osm_id - """ + geometry_type = "point" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom, localrank, scalerank, maki, name FROM layer_poi_label(!BBOX!, !ZOOM!)" [[providers.layers]] name = "road_label" - geometry_type = "LineString" - geometry_fieldname = "way" - fields = [ "len", "name", "osm_id", "ref", "reflen", "shield" ] - sql = """ - SELECT osm_id, 'default' AS shield, ST_AsBinary(way) AS way, name, name_, ref, reflen, len FROM ( - SELECT - osm_id, - way, - name, - (hstore_to_json(extract_names(tags)))::text name_, - CASE - WHEN highway IN ('motorway', 'motorway_link', 'driveway') THEN highway - WHEN highway IN ('primary', 'primary_link', 'trunk', 'trunk_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') THEN 'main' - WHEN highway IN ('residential', 'unclassified', 'living_street') THEN 'street' - WHEN highway IN ('pedestrian', 'construction') OR access = 'private' THEN 'street_limited' - WHEN railway IN ('rail', 'monorail', 'narrow_gauge', 'subway', 'tram') THEN 'major_rail' - WHEN highway IN ('service', 'track') THEN 'service' - WHEN highway IN ('path', 'cycleway', 'ski', 'steps', 'bridleway', 'footway') THEN 'path' - WHEN railway IN ('funicular', 'light_rail', 'preserved') THEN 'minor_rail' - ELSE bail_out('Unexpected road row with osm_id=%s', osm_id::TEXT) - END AS class, - z_order, - CASE - WHEN bridge IS NOT NULL AND bridge <> '' AND bridge <> 'no' AND bridge <> '0' THEN 'bridge' - WHEN tunnel IS NOT NULL AND tunnel <> '' AND tunnel <> 'no' AND tunnel <> '0' THEN 'tunnel' - ELSE 'road' - END AS \"is\", - ref, - pg_catalog.char_length(ref) AS reflen, - ROUND(merc_length(way)) AS len - FROM planet_osm_line - WHERE - ( - ( - highway IN ('motorway', 'primary', 'primary_link', 'trunk', - 'trunk_link', 'secondary', 'secondary_link' - ) - AND ( (name IS NOT NULL AND name <> '') OR (ref IS NOT NULL AND ref <> '')) - AND z(!SCALE_DENOMINATOR!) >= 11 - ) - OR - ( -- 'main' - highway IN ('tertiary', 'tertiary_link', 'residential', 'unclassified', - 'living_street', 'pedestrian', 'construction', 'rail', 'monorail', - 'narrow_gauge', 'subway', 'tram' - ) - AND (name IS NOT NULL AND name <> '') - AND z(!SCALE_DENOMINATOR!) >= 12 - ) - OR - ( -- 'motorway_link' - highway IN ('motorway_link', 'service', 'track', 'driveway', 'path', - 'cycleway', 'ski', 'steps', 'bridleway', 'footway', 'funicular', - 'light_rail', 'preserved' - ) - AND (name IS NOT NULL AND name <> '') - AND z(!SCALE_DENOMINATOR!) >= 14 - ) - ) - --AND linelabel(z(!SCALE_DENOMINATOR!), name, way) - AND way && !BBOX! - ) data JOIN ( - VALUES - ('motorway', 1000), - ('main', 900), - ('street', 800), - ('motorway_link', 700), - ('street_limited', 600), - ('driveway', 500), - ('major_rail', 400), - ('service', 300), - ('minor_rail', 200), - ('path', 100) - ) AS ordertable(feature, prio) ON class=feature - -- Sort the same way as the road layer, so that more important streets go - -- first and have a higher priority to be rendered - ORDER BY z_order + prio + - CASE \"is\" - WHEN 'tunnel' THEN -100000 - WHEN 'road' THEN 0 - WHEN 'bridge' THEN 100000 - ELSE bail_out('Unexpected row with is=%s, osm_id=%s', \"is\", osm_id::TEXT)::INT - END - """ + geometry_type = "linestring" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom, shield, name, name_, ref, reflen, len FROM layer_transportation_name(!BBOX!, !ZOOM!)" [[providers.layers]] name = "place_label" - geometry_type = "Point" - geometry_fieldname = "way" - fields = [ "ldir", "localrank", "name", "osm_id", "type" ] - sql = """ - SELECT - osm_id, - ST_AsBinary(way) AS way, - name, - name_, - \"type\", - ldir, - localrank - FROM( - SELECT - DISTINCT ON (labelgrid(way, 16, !PIXEL_WIDTH!)) - osm_id, - way, - get_label_name(name) AS name, - (hstore_to_json(extract_names(tags)))::text name_, - place AS \"type\", - 'SE' AS ldir, - 1 AS localrank, -- TODO: - CASE - WHEN place = 'city' THEN 5000000000 + to_int(population) - WHEN place = 'town' THEN 3000000000 + to_int(population) - WHEN place = 'village' THEN 1000000000 + to_int(population) - ELSE to_int(population) - END AS sort_order - FROM planet_osm_point - WHERE - ( - ( - place = 'city' - AND z(!SCALE_DENOMINATOR!) >= 4 - -- On zoom 4, display cities with 1M+ population. Decrease by 250k every level - AND (to_int(population) + z(!SCALE_DENOMINATOR!) * 250000 - 2000000) > 0 - ) - OR - ( - place = 'town' - AND z(!SCALE_DENOMINATOR!) >= 9 - ) - OR - ( - place = 'village' - AND z(!SCALE_DENOMINATOR!) >= 11 - ) - OR - ( - place IN ('hamlet', 'suburb','neighbourhood') - AND z(!SCALE_DENOMINATOR!) >= 13 - ) - ) - AND (name IS NOT NULL AND name <> '') - AND way && !BBOX! --ST_Expand(!BBOX!, 64*!PIXEL_WIDTH!) - ORDER BY - labelgrid(way, 16, !PIXEL_WIDTH!), - sort_order DESC, - pg_catalog.length(name) DESC, - name - ) data ORDER BY sort_order DESC - """ + geometry_type = "point" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom, ldir, localrank, name, type FROM layer_place_label(!BBOX!, !ZOOM!, !PIXEL_WIDTH!)" [[providers.layers]] name = "building" - geometry_type = "Polygon" - geometry_fieldname = "way" - fields = [ "osm_id" ] - sql = """ - SELECT osm_id, ST_AsBinary(way) AS way - FROM planet_osm_polygon - WHERE - z(!SCALE_DENOMINATOR!) >= 14 - AND (building IS NOT NULL AND building <> '') - AND building <> 'no' - AND way && !BBOX! - """ + geometry_type = "polygon" + geometry_fieldname = "geom" + id_fieldname = "osm_id" + sql = "SELECT osm_id, ST_AsMVTGeom(geometry, !BBOX!) AS geom FROM layer_buildings(!BBOX!, !ZOOM!)" [[maps]] name = "osm" attribution = 'Map data © OpenStreetMap contributors' -center = [ -122.4144, 37.7907, 14.0 ] -tile_buffer = 8 +center = [ -0.2416797, 51.5287718, 14.0 ] # London, UK [[maps.layers]] provider_layer = "osm.landuse" - min_zoom = 0 + min_zoom = 7 max_zoom = 15 [[maps.layers]] provider_layer = "osm.waterway" - min_zoom = 0 + min_zoom = 8 max_zoom = 15 [[maps.layers]] @@ -553,40 +114,40 @@ tile_buffer = 8 [[maps.layers]] provider_layer = "osm.aeroway" - min_zoom = 0 + min_zoom = 12 max_zoom = 15 [[maps.layers]] - provider_layer = "osm.road" - min_zoom = 0 + provider_layer = "osm.building" + min_zoom = 14 max_zoom = 15 [[maps.layers]] - provider_layer = "osm.admin" - min_zoom = 0 + provider_layer = "osm.road" + min_zoom = 6 max_zoom = 15 [[maps.layers]] - provider_layer = "osm.country_label" + provider_layer = "osm.admin" min_zoom = 0 max_zoom = 15 [[maps.layers]] - provider_layer = "osm.poi_label" - min_zoom = 0 - max_zoom = 15 + provider_layer = "osm.country_label" + min_zoom = 3 + max_zoom = 10 [[maps.layers]] - provider_layer = "osm.road_label" - min_zoom = 0 + provider_layer = "osm.place_label" + min_zoom = 3 max_zoom = 15 [[maps.layers]] - provider_layer = "osm.place_label" - min_zoom = 0 + provider_layer = "osm.poi_label" + min_zoom = 14 max_zoom = 15 [[maps.layers]] - provider_layer = "osm.building" - min_zoom = 0 + provider_layer = "osm.road_label" + min_zoom = 11 max_zoom = 15 diff --git a/workspace/Dockerfile b/workspace/Dockerfile index ae42701..7d742d3 100644 --- a/workspace/Dockerfile +++ b/workspace/Dockerfile @@ -32,8 +32,7 @@ RUN mkdir -p $GOPATH/src/github.com/omniscale/imposm3 \ && rm -rf $GOPATH/src/ \ && rm -rf $GOPATH/pkg/ -COPY ./config.template.json /etc/kartosm/config.template.json -COPY ./mapping.yml /etc/kartosm/mapping.yml +COPY ./config.template.json /etc/imposm/config.template.json COPY osm-initial-import.sh /usr/bin/osm-initial-import RUN chmod +x /usr/bin/osm-initial-import diff --git a/workspace/config.template.json b/workspace/config.template.json index 7ed6526..90bce2d 100644 --- a/workspace/config.template.json +++ b/workspace/config.template.json @@ -1,9 +1,9 @@ { - "cachedir": "/etc/kartosm/cache", - "diffdir": "/etc/kartosm/diff", - "expiretiles_dir": "/etc/kartosm/expiretiles", + "cachedir": "/srv/imposm/cache", + "diffdir": "/srv/imposm/diff", + "expiretiles_dir": "/srv/imposm/expiretiles", "expiretiles_zoom": 15, "connection": "postgis: user=$PGUSER password=$PGPASSWORD dbname=$PGDATABASE host=postgres-postgis prefix=NONE", - "mapping": "/etc/kartosm/mapping.yml", + "mapping": "/etc/imposm/mapping.yml", "replication_url": "https://planet.openstreetmap.org/replication/minute/" } diff --git a/workspace/mapping.yml b/workspace/mapping.yml index 8a38158..76ab323 100644 --- a/workspace/mapping.yml +++ b/workspace/mapping.yml @@ -1,9 +1,45 @@ areas: area_tags: [buildings, landuse, leisure, natural, aeroway, amenity] - linear_tags: [highway, barrier, boundary] + linear_tags: [highway, barrier] + tags: load_all: true exclude: [note, "note:*", source, source_ref, "source:*", attribution, comment, fixme, created_by, odbl, "odbl:note", "SK53_bulk:load", "tiger:*", "NHD:*", "nhd:*", "gnis:*", "geobase:*", "accuracy:meters", "sub_sea:type", "waterway:type", "KSJ2:*", "yh:*", "osak:*", "kms:*", "ngbe:*", "naptan:*", "CLC:*", "3dshapes:ggmodelk", AND_nosr_r, import, "it:fvg:*"] + +generalized_tables: + planet_osm_polygon_landuse_gen_z6: + source: planet_osm_polygon_landuse_gen_z7 + tolerance: 2445.98490513 + sql_filter: area>power(2445.98490513,2) + planet_osm_polygon_landuse_gen_z7: + source: planet_osm_polygon_landuse_gen_z8 + tolerance: 1222.99245256 + sql_filter: area>power(2445.98490513,2) + planet_osm_polygon_landuse_gen_z8: + source: planet_osm_polygon_landuse_gen_z9 + sql_filter: area>power(2445.98490513,2) + tolerance: 611.496226281 + planet_osm_polygon_landuse_gen_z9: + source: planet_osm_polygon_landuse_gen_z10 + sql_filter: area>power(1222.99245256,2) + tolerance: 305.748113141 + planet_osm_polygon_landuse_gen_z10: + source: planet_osm_polygon_landuse_gen_z11 + sql_filter: area>power(611.496226281,2) + tolerance: 152.87405657 + planet_osm_polygon_landuse_gen_z11: + source: planet_osm_polygon_landuse_gen_z12 + sql_filter: area>power(305.748113141,2) + tolerance: 76.4370282852 + planet_osm_polygon_landuse_gen_z12: + source: planet_osm_polygon_landuse_gen_z13 + sql_filter: area>power(152.87405657,2) + tolerance: 38.2185141426 + planet_osm_polygon_landuse_gen_z13: + source: planet_osm_polygon_landuse + sql_filter: area>power(76.4370282852,2) AND ST_IsValid(way) + tolerance: 19.1092570713 + tables: planet_osm_point: type: point @@ -37,7 +73,8 @@ tables: name: tags type: hstore_tags - name: way - type: geometry + type: validated_geometry + planet_osm_polygon: type: polygon mapping: @@ -143,7 +180,114 @@ tables: name: tags type: hstore_tags - name: way - type: geometry + type: validated_geometry + + planet_osm_polygon_landuse: + type: polygon + columns: + - name: osm_id + type: id + - key: aeroway + name: aeroway + type: string + - key: amenity + name: amenity + type: string + - key: area + name: area + type: area + - key: boundary + name: boundary + type: string + - key: landuse + name: landuse + type: string + - key: leisure + name: leisure + type: string + - key: name + name: name + type: string + - key: natural + name: natural + type: string + - name: z_order + type: enumerate + args: + values: + - land + - island + - lan + - heath + - railway + - industrial + - commercial + - retail + - residential + - quarry + - zoo + - vineyard + - orchard + - scrub + - hospital + - place_of_worship + - theatre + - cinema + - nature_reserve + - parking + - fuel + - baracks + - library + - college + - school + - university + - golf_course + - allotments + - common + - pitch + - sports_centre + - garden + - recreation_ground + - village_green + - wetland + - grass + - meadow + - wood + - farmland + - farm + - farmyard + - cemetery + - forest + - park + - playground + - footway + - pedestrian + - water + - name: way + type: validated_geometry + mapping: + natural: + - wood + leisure: + - national_reserve + - nature_reserve + - golf_course + - park + - playground + aeroway: [__any__] + landuse: + - wood + - forest + - cemetery + - industrial + - village_green + amenity: + - school + - university + - hospital + boundary: + - national_park + planet_osm_line: type: linestring mapping: @@ -155,7 +299,7 @@ tables: name: tags type: hstore_tags - name: way - type: geometry + type: validated_geometry - key: aeroway name: aeroway type: string @@ -186,6 +330,7 @@ tables: - key: z_order name: z_order type: wayzorder + admin: type: relation_member mapping: @@ -197,7 +342,7 @@ tables: type: id from_member: true - name: way - type: geometry + type: validated_geometry - key: name name: name type: string @@ -223,4 +368,3 @@ tables: filters: require: admin_level: ['2', '4'] - diff --git a/workspace/osm-initial-import.sh b/workspace/osm-initial-import.sh index 4c3d7eb..62603ae 100644 --- a/workspace/osm-initial-import.sh +++ b/workspace/osm-initial-import.sh @@ -5,15 +5,16 @@ set -e pbf_dir="/srv/pbf" kartotherian_dir="/srv/kartotherian/packages/kartotherian" postgis_vt_util_sql_lib="${kartotherian_dir}/node_modules/@kartotherian/postgis-vt-util/lib.sql" -modules_with_sql="@kartotherian/osm-bright-source @wikimedia/kartotherian-geoshapes" +modules_with_sql="@kartotherian/osm-bright-source/sql/helpers @wikimedia/kartotherian-geoshapes/sql" +osm_bright_source="@kartotherian/osm-bright-source" database_host="postgres-postgis" log_file=/var/log/osm-initial-import.log pbf_file_url= scripts_only=false exec_water_polygons=true # Prepare config.json -cp /etc/kartosm/config.template.json /etc/kartosm/config.json && \ - perl -pe 's/\$([_A-Z]+)/$ENV{$1}/g' -i /etc/kartosm/config.json +cp /etc/imposm/config.template.json /etc/imposm/config.json && \ + perl -pe 's/\$([_A-Z]+)/$ENV{$1}/g' -i /etc/imposm/config.json function show_help() { echo "osm-initial-import -p [-H ] [-s]" @@ -71,7 +72,7 @@ function download_pbf() { function reset_postgres() { echo "starting reset of prosgresql database" - psql -h ${database_host} -U ${PGUSER} -d ${PGDATABASE} -c 'DROP TABLE IF EXISTS admin, planet_osm_line, planet_osm_point, planet_osm_polygon, planet_osm_roads, water_polygons CASCADE;' + psql -h ${database_host} -U ${PGUSER} -d ${PGDATABASE} -c 'DROP TABLE IF EXISTS admin, planet_osm_line, planet_osm_point, planet_osm_polygon, planet_osm_roads, water_polygons, water_polygons_simplified CASCADE;' echo "reset of prosgresql database completed" } @@ -79,7 +80,7 @@ function initial_osm_import() { echo "starting initial OSM import" psql -h ${database_host} -U ${PGUSER} -d ${PGDATABASE} -c 'CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS hstore;' && \ imposm import \ - -config /etc/kartosm/config.json \ + -config /etc/imposm/config.json \ -overwritecache \ -read ${pbf_dir}/${filename} \ -diff \ @@ -91,7 +92,7 @@ function initial_osm_import() { fi echo "initial OSM import completed, starting production deploy" imposm import \ - -config /etc/kartosm/config.json \ + -config /etc/imposm/config.json \ -deployproduction if [ ${PIPESTATUS[0]} -ne 0 ]; then @@ -105,6 +106,7 @@ function import_water_lines() { echo "starting water line import" echo "WARNING: water-polygons-split-3857.zip size is 539M" cd ${pbf_dir} + # import water_polygons if [ ! -f "$pbf_dir/water-polygons-split-3857.zip" ]; then curl -O https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip fi @@ -112,6 +114,14 @@ function import_water_lines() { unzip water-polygons-split-3857.zip fi shp2pgsql -c -s 3857 -g way water-polygons-split-3857/water_polygons.shp water_polygons | psql -h ${database_host} -U ${PGUSER} -d ${PGDATABASE} + # import water_polygons_simplified + if [ ! -f "$pbf_dir/simplified-water-polygons-split-3857.zip" ]; then + curl -O https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip + fi + if [ ! -f "$pbf_dir/simplified-water-polygons-split-3857/simplified_water_polygons.shp" ]; then + unzip simplified-water-polygons-split-3857.zip + fi + shp2pgsql -c -s 3857 -g way simplified-water-polygons-split-3857/simplified_water_polygons.shp water_polygons_simplified | psql -h ${database_host} -U ${PGUSER} -d ${PGDATABASE} echo "water line import completed" } @@ -120,8 +130,8 @@ function custom_functions_and_indexes() { psql -h ${database_host} -U ${PGUSER} -Xd ${PGDATABASE} -f ${postgis_vt_util_sql_lib} cd ${kartotherian_dir} for module in ${modules_with_sql}; do - echo "executing SQL in: ${kartotherian_dir}/node_modules/${module}/sql" - for sql_file in `ls ${kartotherian_dir}/node_modules/${module}/sql/*.sql`; do + echo "executing SQL in: ${kartotherian_dir}/node_modules/${module}" + for sql_file in `ls ${kartotherian_dir}/node_modules/${module}/*.sql`; do echo " executing: ${sql_file}" psql -h ${database_host} -U ${PGUSER} -Xd ${PGDATABASE} -f ${sql_file} done @@ -129,12 +139,24 @@ function custom_functions_and_indexes() { echo "creation of custom functions and indexes completed" } + +function styles_functions() { + echo "starting creation of styles functions" + for sql_file in `ls ${kartotherian_dir}/node_modules/${osm_bright_source}/sql/layers/*.sql`; do + echo " executing: ${sql_file}" + psql -h ${database_host} -U ${PGUSER} -Xd ${PGDATABASE} -f ${sql_file} + done + echo "creation of styles functions completed" +} + function cleanup() { echo "starting cleanup" rm "${pbf_dir}/${filename}.md5" rm "${pbf_dir}/${filename}" - rm "${pbf_dir}/water-polygons-split-4326.zip" - rm -rf "${pbf_dir}/water-polygons-split-4326" + rm "${pbf_dir}/water-polygons-split-3857.zip" + rm -rf "${pbf_dir}/water-polygons-split-3857" + rm "${pbf_dir}/simplified-water-polygons-split-3857.zip" + rm -rf "${pbf_dir}/simplified-water-polygons-split-3857" echo "cleanup completed" } @@ -147,4 +169,5 @@ if [ "$exec_water_polygons" = true ]; then import_water_lines fi custom_functions_and_indexes +styles_functions #cleanup