From 6e78c75da0f93889aeb1b49e7ff31535016a8e11 Mon Sep 17 00:00:00 2001 From: Gabriel Landais Date: Fri, 6 Nov 2020 08:55:51 +0100 Subject: [PATCH 1/7] feature: add Europe as a project --- projects/europe/.env | 3 + projects/europe/README.md | 32 +++++ projects/europe/docker-compose.yml | 112 +++++++++++++++ projects/europe/elasticsearch.yml | 4 + projects/europe/pelias.json | 213 +++++++++++++++++++++++++++++ 5 files changed, 364 insertions(+) create mode 100644 projects/europe/.env create mode 100644 projects/europe/README.md create mode 100644 projects/europe/docker-compose.yml create mode 100644 projects/europe/elasticsearch.yml create mode 100644 projects/europe/pelias.json diff --git a/projects/europe/.env b/projects/europe/.env new file mode 100644 index 00000000..41889a07 --- /dev/null +++ b/projects/europe/.env @@ -0,0 +1,3 @@ +COMPOSE_PROJECT_NAME=pelias +DATA_DIR=/tmp/pelias/europe +ENABLE_GEONAMES=false diff --git a/projects/europe/README.md b/projects/europe/README.md new file mode 100644 index 00000000..acb5f8a3 --- /dev/null +++ b/projects/europe/README.md @@ -0,0 +1,32 @@ +# Europe area + +This project is configured to download/prepare/build a complete Pelias installation for Europe. + +# Setup + +Please refer to the instructions at in order to install and configure your docker environment. + +The minimum configuration required in order to run this project are [installing prerequisites](https://github.com/pelias/docker#prerequisites), [install the pelias command](https://github.com/pelias/docker#installing-the-pelias-command) and [configure the environment](https://github.com/pelias/docker#configure-environment). + +Please ensure that's all working fine before continuing. + +# Run a Build + +To run a complete build, execute the following commands: + +```bash +pelias compose pull +pelias elastic start +pelias elastic wait +pelias elastic create +pelias download all +pelias prepare all +pelias import all +pelias compose up +``` + +# Make an Example Query + +You can now make queries against your new Pelias build: + + diff --git a/projects/europe/docker-compose.yml b/projects/europe/docker-compose.yml new file mode 100644 index 00000000..8ffc4b65 --- /dev/null +++ b/projects/europe/docker-compose.yml @@ -0,0 +1,112 @@ +version: '3' +networks: + default: + driver: bridge +services: + libpostal: + image: pelias/libpostal-service + container_name: pelias_libpostal + user: "${DOCKER_USER}" + restart: always + ports: [ "4400:4400" ] + schema: + image: pelias/schema:master + container_name: pelias_schema + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + api: + image: pelias/api:master + container_name: pelias_api + user: "${DOCKER_USER}" + restart: always + environment: [ "PORT=4000" ] + ports: [ "4000:4000" ] + volumes: + - "./pelias.json:/code/pelias.json" + placeholder: + image: pelias/placeholder:master + container_name: pelias_placeholder + user: "${DOCKER_USER}" + restart: always + environment: [ "PORT=4100" ] + ports: [ "4100:4100" ] + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + - "./blacklist/:/data/blacklist" + whosonfirst: + image: pelias/whosonfirst:master + container_name: pelias_whosonfirst + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + openstreetmap: + image: pelias/openstreetmap:master + container_name: pelias_openstreetmap + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + openaddresses: + image: pelias/openaddresses:master + container_name: pelias_openaddresses + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + csv-importer: + image: pelias/csv-importer:master + container_name: pelias_csv_importer + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + - "./blacklist/:/data/blacklist" + polylines: + image: pelias/polylines:master + container_name: pelias_polylines + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + interpolation: + image: pelias/interpolation:master + container_name: pelias_interpolation + user: "${DOCKER_USER}" + restart: always + environment: [ "PORT=4300" ] + ports: [ "4300:4300" ] + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + pip: + image: pelias/pip-service:master + container_name: pelias_pip-service + user: "${DOCKER_USER}" + restart: always + environment: [ "PORT=4200" ] + ports: [ "4200:4200" ] + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + elasticsearch: + image: pelias/elasticsearch:7.5.1 + container_name: pelias_elasticsearch + user: "${DOCKER_USER}" + restart: always + ports: [ "9200:9200", "9300:9300" ] + volumes: + - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro" + - "${DATA_DIR}/elasticsearch:/usr/share/elasticsearch/data" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + cap_add: [ "IPC_LOCK" ] + security_opt: + - seccomp=unconfined diff --git a/projects/europe/elasticsearch.yml b/projects/europe/elasticsearch.yml new file mode 100644 index 00000000..ca137940 --- /dev/null +++ b/projects/europe/elasticsearch.yml @@ -0,0 +1,4 @@ +network.host: 0.0.0.0 +bootstrap.memory_lock: true +indices.breaker.fielddata.limit: 85% +indices.fielddata.cache.size: 75% \ No newline at end of file diff --git a/projects/europe/pelias.json b/projects/europe/pelias.json new file mode 100644 index 00000000..97499c3c --- /dev/null +++ b/projects/europe/pelias.json @@ -0,0 +1,213 @@ +{ + "logger": { + "level": "info", + "timestamp": false + }, + "esclient": { + "apiVersion": "7.5", + "hosts": [ + { "host": "elasticsearch" } + ] + }, + "elasticsearch": { + "settings": { + "index": { + "refresh_interval": "10s", + "number_of_replicas": "0", + "number_of_shards": "4" + } + } + }, + "api": { + "services": { + "pip": { "url": "http://pip:4200" }, + "libpostal": { "url": "http://libpostal:4400" }, + "placeholder": { "url": "http://placeholder:4100" }, + "interpolation": { "url": "http://interpolation:4300" } + } + }, + "imports": { + "adminLookup": { + "enabled": true + }, + "openstreetmap": { + "download": [ + { "sourceURL": "https://download.geofabrik.de/europe-latest.osm.pbf" } + ], + "leveldbpath": "/tmp", + "datapath": "/data/openstreetmap", + "import": [{ + "filename": "europe-latest.osm.pbf" + }] + }, + "openaddresses": { + "datapath": "/data/openaddresses", + "files": [ + "be/bru/bosa-region-brussels-fr.csv", + "be/bru/bosa-region-brussels-nl.csv", + "be/vlg/bosa-region-flanders-fr.csv", + "be/vlg/bosa-region-flanders-nl.csv", + "be/wal/bosa-region-wallonia-de.csv", + "be/wal/bosa-region-wallonia-fr.csv", + "cz/countrywide.csv", + "dk/countrywide.csv", + "de/berlin.csv", + "de/he/city_of_frankfurtammain.csv", + "de/he/city_of_kassel.csv", + "de/hh/statewide.csv", + "de/mv/city_of_rostock.csv", + "de/ni/bentheim.csv", + "de/nw/city_of_cologne.csv", + "de/nw/city_of_wesel.csv", + "de/nw/city_of_wuppertal.csv", + "de/nw/kreis_viersen.csv", + "de/nw/statewide.csv", + "de/sn/statewide.csv", + "de/th/statewide.csv", + "ee/countrywide.csv", + "es/25829.csv", + "es/25830.csv", + "es/25831.csv", + "es/32628.csv", + "es/nc/statewide.csv", + "fr/countrywide.csv", + "it/04/acerra.csv", + "it/21/city_of_biella.csv", + "it/21/statewide.csv", + "it/21/torino.csv", + "it/25/bg.csv", + "it/25/bs.csv", + "it/25/co.csv", + "it/25/cr.csv", + "it/25/lc.csv", + "it/25/lo.csv", + "it/25/mb.csv", + "it/25/mi.csv", + "it/25/mn.csv", + "it/25/pv.csv", + "it/25/so.csv", + "it/25/va.csv", + "it/32/south-tyrol-de.csv", + "it/32/south-tyrol-it.csv", + "it/32/south-tyrol-lld.csv", + "it/34/city_of_verona.csv", + "it/34/venice.csv", + "it/36/statewide.csv", + "it/42/statewide.csv", + "it/45/bo.csv", + "it/45/bologna.csv", + "it/45/fc.csv", + "it/45/fe.csv", + "it/45/ferrara.csv", + "it/45/mo.csv", + "it/45/pc.csv", + "it/45/pr.csv", + "it/45/ra.csv", + "it/45/re.csv", + "it/45/rn.csv", + "it/52/firenze.csv", + "it/52/statewide.csv", + "it/55/statewide.csv", + "it/82/statewide.csv", + "it/88/statewide.csv", + "it/countrywide.csv", + "lv/city_of_riga.csv", + "lt/city_of_vilnius.csv", + "lt/countrywide.csv", + "lu/countrywide.csv", + "nl/countrywide.csv", + "at/31254.csv", + "at/31255.csv", + "at/31256.csv", + "at/city_of_vienna.csv", + "at/tirol.csv", + "pl/dolnoslaskie.csv", + "pl/kujawsko-pomorskie.csv", + "pl/lodzkie.csv", + "pl/lubelskie.csv", + "pl/lubuskie.csv", + "pl/malopolskie.csv", + "pl/mazowieckie.csv", + "pl/opolskie.csv", + "pl/podkarpackie.csv", + "pl/podlaskie.csv", + "pl/pomorskie.csv", + "pl/slaskie.csv", + "pl/swietokrzyskie.csv", + "pl/warminsko-mazurskie.csv", + "pl/wielkopolskie.csv", + "pl/zachodniopomorskie.csv", + "pt/countrywide.csv", + "ro/bucharest-metro.csv", + "ro/bucharest.csv", + "si/countrywide.csv", + "sk/countrywide.csv", + "fi/ahvenanmaa-fi.csv", + "fi/ahvenanmaa-sv.csv", + "fi/countrywide-fi.csv", + "fi/countrywide-sv.csv", + "fi/etelä-karjala-fi.csv", + "fi/etelä-karjala-sv.csv", + "fi/etelä-pohjanmaa-fi.csv", + "fi/etelä-pohjanmaa-sv.csv", + "fi/etelä-savo-fi.csv", + "fi/etelä-savo-sv.csv", + "fi/kainuu-fi.csv", + "fi/kainuu-sv.csv", + "fi/kanta-häme-fi.csv", + "fi/kanta-häme-sv.csv", + "fi/keski-pohjanmaa-fi.csv", + "fi/keski-pohjanmaa-sv.csv", + "fi/keski-suomi-fi.csv", + "fi/keski-suomi-sv.csv", + "fi/kymeenlaakso-fi.csv", + "fi/kymeenlaakso-sv.csv", + "fi/lappi-fi.csv", + "fi/lappi-sv.csv", + "fi/päijät-häme-fi.csv", + "fi/päijät-häme-sv.csv", + "fi/pirkanmaa-fi.csv", + "fi/pirkanmaa-sv.csv", + "fi/pohjanmaa-fi.csv", + "fi/pohjanmaa-sv.csv", + "fi/pohjois-karjala-fi.csv", + "fi/pohjois-karjala-sv.csv", + "fi/pohjois-pohjanmaa-fi.csv", + "fi/pohjois-pohjanmaa-sv.csv", + "fi/pohjois-savo-fi.csv", + "fi/pohjois-savo-sv.csv", + "fi/satakunta-fi.csv", + "fi/satakunta-sv.csv", + "fi/uusimaa-fi.csv", + "fi/uusimaa-sv.csv", + "fi/varsinais-suomi-fi.csv", + "fi/varsinais-suomi-sv.csv", + "se/municipality_of_alingsas.csv", + "se/municipality_of_gislaved.csv", + "se/municipality_of_goteborg.csv", + "se/municipality_of_helsingborg.csv", + "se/municipality_of_hoganas.csv", + "se/municipality_of_kalmar.csv", + "se/municipality_of_kristinehamn.csv", + "se/municipality_of_malmo.csv", + "se/municipality_of_nacka.csv", + "se/municipality_of_Österåker.csv", + "se/municipality_of_savsjö.csv", + "se/municipality_of_stockholm.csv", + "se/municipality_of_uppsala.csv", + "se/municipality_of_vasteras.csv", + "se/municipality_of_vaxholm.csv", + "se/municipality_of_vaxjo.csv" + ] + }, + "polyline": { + "datapath": "/data/polylines", + "files": [ "extract.0sv" ] + }, + "whosonfirst": { + "datapath": "/data/whosonfirst", + "importPostalcodes": true, + "importPlace": [ "102191581" ] + } + } +} From 9ee55161c37c434a345cf9131d782ef4d0e6692a Mon Sep 17 00:00:00 2001 From: Gabriel Landais Date: Wed, 9 Dec 2020 18:55:34 +0100 Subject: [PATCH 2/7] Europe settings Geonames (ALL as we can't select countries) Tiger interpolation --- projects/europe/pelias.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/europe/pelias.json b/projects/europe/pelias.json index 97499c3c..6579abce 100644 --- a/projects/europe/pelias.json +++ b/projects/europe/pelias.json @@ -14,7 +14,7 @@ "index": { "refresh_interval": "10s", "number_of_replicas": "0", - "number_of_shards": "4" + "number_of_shards": "1" } } }, @@ -30,6 +30,10 @@ "adminLookup": { "enabled": true }, + "geonames": { + "datapath": "/data/geonames", + "countryCode": "ALL" + }, "openstreetmap": { "download": [ { "sourceURL": "https://download.geofabrik.de/europe-latest.osm.pbf" } @@ -208,6 +212,13 @@ "datapath": "/data/whosonfirst", "importPostalcodes": true, "importPlace": [ "102191581" ] + }, + "interpolation": { + "download": { + "tiger": { + "datapath": "/data/tiger" + } + } } } } From d500f251e3fe19ab938defc039d7010ca8ef25a2 Mon Sep 17 00:00:00 2001 From: Gabriel Landais Date: Wed, 9 Dec 2020 19:00:07 +0100 Subject: [PATCH 3/7] Europe : enable geonames --- projects/europe/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/europe/.env b/projects/europe/.env index 41889a07..72e82de8 100644 --- a/projects/europe/.env +++ b/projects/europe/.env @@ -1,3 +1,3 @@ COMPOSE_PROJECT_NAME=pelias DATA_DIR=/tmp/pelias/europe -ENABLE_GEONAMES=false +ENABLE_GEONAMES=true From 28093828c332cfb7c253ed4c0626731fee2ded56 Mon Sep 17 00:00:00 2001 From: Gabriel Landais Date: Wed, 9 Dec 2020 19:08:51 +0100 Subject: [PATCH 4/7] Europe : remove Tiger download (USA) --- projects/europe/pelias.json | 7 ------- 1 file changed, 7 deletions(-) diff --git a/projects/europe/pelias.json b/projects/europe/pelias.json index 6579abce..66307683 100644 --- a/projects/europe/pelias.json +++ b/projects/europe/pelias.json @@ -212,13 +212,6 @@ "datapath": "/data/whosonfirst", "importPostalcodes": true, "importPlace": [ "102191581" ] - }, - "interpolation": { - "download": { - "tiger": { - "datapath": "/data/tiger" - } - } } } } From 8b471c7e372bd7884671ad537ff77a5940475143 Mon Sep 17 00:00:00 2001 From: Gabriel Landais Date: Thu, 10 Dec 2020 08:21:46 +0100 Subject: [PATCH 5/7] Replace openaddresses import with custom import Too many files to download, some of them fail often --- projects/europe/README.md | 12 +++ projects/europe/pelias.json | 155 ------------------------------------ 2 files changed, 12 insertions(+), 155 deletions(-) diff --git a/projects/europe/README.md b/projects/europe/README.md index acb5f8a3..0c851fc4 100644 --- a/projects/europe/README.md +++ b/projects/europe/README.md @@ -20,6 +20,18 @@ pelias elastic start pelias elastic wait pelias elastic create pelias download all +``` + +Download OpenAddresses full Europe : +```bash +data_dir=... +mkdir ${data_dir}/openaddresses +curl -s -L -X GET -o /tmp/europe.zip https://data.openaddresses.io/openaddr-collected-europe.zip +unzip -o -qq -d ${data_dir}/openaddresses /tmp/europe.zip +rm /tmp/europe.zip +``` + +```bash pelias prepare all pelias import all pelias compose up diff --git a/projects/europe/pelias.json b/projects/europe/pelias.json index 66307683..dc100491 100644 --- a/projects/europe/pelias.json +++ b/projects/europe/pelias.json @@ -47,161 +47,6 @@ "openaddresses": { "datapath": "/data/openaddresses", "files": [ - "be/bru/bosa-region-brussels-fr.csv", - "be/bru/bosa-region-brussels-nl.csv", - "be/vlg/bosa-region-flanders-fr.csv", - "be/vlg/bosa-region-flanders-nl.csv", - "be/wal/bosa-region-wallonia-de.csv", - "be/wal/bosa-region-wallonia-fr.csv", - "cz/countrywide.csv", - "dk/countrywide.csv", - "de/berlin.csv", - "de/he/city_of_frankfurtammain.csv", - "de/he/city_of_kassel.csv", - "de/hh/statewide.csv", - "de/mv/city_of_rostock.csv", - "de/ni/bentheim.csv", - "de/nw/city_of_cologne.csv", - "de/nw/city_of_wesel.csv", - "de/nw/city_of_wuppertal.csv", - "de/nw/kreis_viersen.csv", - "de/nw/statewide.csv", - "de/sn/statewide.csv", - "de/th/statewide.csv", - "ee/countrywide.csv", - "es/25829.csv", - "es/25830.csv", - "es/25831.csv", - "es/32628.csv", - "es/nc/statewide.csv", - "fr/countrywide.csv", - "it/04/acerra.csv", - "it/21/city_of_biella.csv", - "it/21/statewide.csv", - "it/21/torino.csv", - "it/25/bg.csv", - "it/25/bs.csv", - "it/25/co.csv", - "it/25/cr.csv", - "it/25/lc.csv", - "it/25/lo.csv", - "it/25/mb.csv", - "it/25/mi.csv", - "it/25/mn.csv", - "it/25/pv.csv", - "it/25/so.csv", - "it/25/va.csv", - "it/32/south-tyrol-de.csv", - "it/32/south-tyrol-it.csv", - "it/32/south-tyrol-lld.csv", - "it/34/city_of_verona.csv", - "it/34/venice.csv", - "it/36/statewide.csv", - "it/42/statewide.csv", - "it/45/bo.csv", - "it/45/bologna.csv", - "it/45/fc.csv", - "it/45/fe.csv", - "it/45/ferrara.csv", - "it/45/mo.csv", - "it/45/pc.csv", - "it/45/pr.csv", - "it/45/ra.csv", - "it/45/re.csv", - "it/45/rn.csv", - "it/52/firenze.csv", - "it/52/statewide.csv", - "it/55/statewide.csv", - "it/82/statewide.csv", - "it/88/statewide.csv", - "it/countrywide.csv", - "lv/city_of_riga.csv", - "lt/city_of_vilnius.csv", - "lt/countrywide.csv", - "lu/countrywide.csv", - "nl/countrywide.csv", - "at/31254.csv", - "at/31255.csv", - "at/31256.csv", - "at/city_of_vienna.csv", - "at/tirol.csv", - "pl/dolnoslaskie.csv", - "pl/kujawsko-pomorskie.csv", - "pl/lodzkie.csv", - "pl/lubelskie.csv", - "pl/lubuskie.csv", - "pl/malopolskie.csv", - "pl/mazowieckie.csv", - "pl/opolskie.csv", - "pl/podkarpackie.csv", - "pl/podlaskie.csv", - "pl/pomorskie.csv", - "pl/slaskie.csv", - "pl/swietokrzyskie.csv", - "pl/warminsko-mazurskie.csv", - "pl/wielkopolskie.csv", - "pl/zachodniopomorskie.csv", - "pt/countrywide.csv", - "ro/bucharest-metro.csv", - "ro/bucharest.csv", - "si/countrywide.csv", - "sk/countrywide.csv", - "fi/ahvenanmaa-fi.csv", - "fi/ahvenanmaa-sv.csv", - "fi/countrywide-fi.csv", - "fi/countrywide-sv.csv", - "fi/etelä-karjala-fi.csv", - "fi/etelä-karjala-sv.csv", - "fi/etelä-pohjanmaa-fi.csv", - "fi/etelä-pohjanmaa-sv.csv", - "fi/etelä-savo-fi.csv", - "fi/etelä-savo-sv.csv", - "fi/kainuu-fi.csv", - "fi/kainuu-sv.csv", - "fi/kanta-häme-fi.csv", - "fi/kanta-häme-sv.csv", - "fi/keski-pohjanmaa-fi.csv", - "fi/keski-pohjanmaa-sv.csv", - "fi/keski-suomi-fi.csv", - "fi/keski-suomi-sv.csv", - "fi/kymeenlaakso-fi.csv", - "fi/kymeenlaakso-sv.csv", - "fi/lappi-fi.csv", - "fi/lappi-sv.csv", - "fi/päijät-häme-fi.csv", - "fi/päijät-häme-sv.csv", - "fi/pirkanmaa-fi.csv", - "fi/pirkanmaa-sv.csv", - "fi/pohjanmaa-fi.csv", - "fi/pohjanmaa-sv.csv", - "fi/pohjois-karjala-fi.csv", - "fi/pohjois-karjala-sv.csv", - "fi/pohjois-pohjanmaa-fi.csv", - "fi/pohjois-pohjanmaa-sv.csv", - "fi/pohjois-savo-fi.csv", - "fi/pohjois-savo-sv.csv", - "fi/satakunta-fi.csv", - "fi/satakunta-sv.csv", - "fi/uusimaa-fi.csv", - "fi/uusimaa-sv.csv", - "fi/varsinais-suomi-fi.csv", - "fi/varsinais-suomi-sv.csv", - "se/municipality_of_alingsas.csv", - "se/municipality_of_gislaved.csv", - "se/municipality_of_goteborg.csv", - "se/municipality_of_helsingborg.csv", - "se/municipality_of_hoganas.csv", - "se/municipality_of_kalmar.csv", - "se/municipality_of_kristinehamn.csv", - "se/municipality_of_malmo.csv", - "se/municipality_of_nacka.csv", - "se/municipality_of_Österåker.csv", - "se/municipality_of_savsjö.csv", - "se/municipality_of_stockholm.csv", - "se/municipality_of_uppsala.csv", - "se/municipality_of_vasteras.csv", - "se/municipality_of_vaxholm.csv", - "se/municipality_of_vaxjo.csv" ] }, "polyline": { From b661c3764e5097bf68b11f178da6f75c1cf72c73 Mon Sep 17 00:00:00 2001 From: Gabriel Landais Date: Thu, 10 Dec 2020 08:48:53 +0100 Subject: [PATCH 6/7] Add a single openaddress to not download planet --- projects/europe/pelias.json | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/europe/pelias.json b/projects/europe/pelias.json index dc100491..9d2d80e3 100644 --- a/projects/europe/pelias.json +++ b/projects/europe/pelias.json @@ -47,6 +47,7 @@ "openaddresses": { "datapath": "/data/openaddresses", "files": [ + "de/berlin.csv" ] }, "polyline": { From dea23cf833db0e0f23a2b109c8283c9363f68ce4 Mon Sep 17 00:00:00 2001 From: Gabriel Landais Date: Thu, 10 Dec 2020 18:46:40 +0100 Subject: [PATCH 7/7] Hack for importing all OA files --- projects/europe/README.md | 2 + .../{pelias.json => pelias-download.json} | 0 projects/europe/pelias-prepare-import.json | 173 ++++++++++++++++++ 3 files changed, 175 insertions(+) rename projects/europe/{pelias.json => pelias-download.json} (100%) create mode 100644 projects/europe/pelias-prepare-import.json diff --git a/projects/europe/README.md b/projects/europe/README.md index 0c851fc4..6d82ca5f 100644 --- a/projects/europe/README.md +++ b/projects/europe/README.md @@ -15,6 +15,7 @@ Please ensure that's all working fine before continuing. To run a complete build, execute the following commands: ```bash +cp pelias-download.json pelias.json pelias compose pull pelias elastic start pelias elastic wait @@ -32,6 +33,7 @@ rm /tmp/europe.zip ``` ```bash +cp pelias-prepare-import.json pelias.json pelias prepare all pelias import all pelias compose up diff --git a/projects/europe/pelias.json b/projects/europe/pelias-download.json similarity index 100% rename from projects/europe/pelias.json rename to projects/europe/pelias-download.json diff --git a/projects/europe/pelias-prepare-import.json b/projects/europe/pelias-prepare-import.json new file mode 100644 index 00000000..0648034e --- /dev/null +++ b/projects/europe/pelias-prepare-import.json @@ -0,0 +1,173 @@ +{ + "logger": { + "level": "info", + "timestamp": false + }, + "esclient": { + "apiVersion": "7.5", + "hosts": [ + { "host": "elasticsearch" } + ] + }, + "elasticsearch": { + "settings": { + "index": { + "refresh_interval": "10s", + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + }, + "api": { + "services": { + "pip": { "url": "http://pip:4200" }, + "libpostal": { "url": "http://libpostal:4400" }, + "placeholder": { "url": "http://placeholder:4100" }, + "interpolation": { "url": "http://interpolation:4300" } + } + }, + "imports": { + "adminLookup": { + "enabled": true + }, + "geonames": { + "datapath": "/data/geonames", + "countryCode": "ALL" + }, + "openstreetmap": { + "download": [ + { "sourceURL": "https://download.geofabrik.de/europe-latest.osm.pbf" } + ], + "leveldbpath": "/tmp", + "datapath": "/data/openstreetmap", + "import": [{ + "filename": "europe-latest.osm.pbf" + }] + }, + "openaddresses": { + "datapath": "/data/openaddresses", + "files": [ + "de/th/statewide.csv", + "de/sn/statewide.csv", + "de/berlin.csv", + "de/nw/statewide.csv", + "de/nw/city_of_wuppertal.csv", + "de/nw/city_of_wesel.csv", + "de/nw/kreis_viersen.csv", + "de/nw/city_of_cologne.csv", + "de/he/city_of_frankfurtammain.csv", + "de/ni/bentheim.csv", + "de/hh/statewide.csv", + "ro/bucharest-metro.csv", + "ro/bucharest.csv", + "pl/warminsko-mazurskie.csv", + "pl/slaskie.csv", + "pl/podlaskie.csv", + "pl/malopolskie.csv", + "pl/mazowieckie.csv", + "pl/dolnoslaskie.csv", + "pl/wielkopolskie.csv", + "pl/podkarpackie.csv", + "pl/lubelskie.csv", + "pl/lodzkie.csv", + "pl/swietokrzyskie.csv", + "pl/kujawsko-pomorskie.csv", + "pl/zachodniopomorskie.csv", + "pl/opolskie.csv", + "pl/lubuskie.csv", + "pl/pomorskie.csv", + "sk/countrywide.csv", + "si/countrywide.csv", + "fi/countrywide-sv.csv", + "fi/countrywide-fi.csv", + "lv/city_of_riga.csv", + "fr/countrywide.csv", + "it/34/city_of_verona.csv", + "it/34/venice.csv", + "it/21/torino.csv", + "it/21/statewide.csv", + "it/21/city_of_biella.csv", + "it/55/statewide.csv", + "it/04/acerra.csv", + "it/88/statewide.csv", + "it/42/statewide.csv", + "it/45/rn.csv", + "it/45/ferrara.csv", + "it/45/mo.csv", + "it/45/bo.csv", + "it/45/ra.csv", + "it/45/fe.csv", + "it/45/pr.csv", + "it/45/re.csv", + "it/45/fc.csv", + "it/45/pc.csv", + "it/45/bologna.csv", + "it/32/south-tyrol-lld.csv", + "it/32/south-tyrol-it.csv", + "it/32/south-tyrol-de.csv", + "it/36/statewide.csv", + "it/25/mb.csv", + "it/25/bs.csv", + "it/25/co.csv", + "it/25/bg.csv", + "it/25/lc.csv", + "it/25/cr.csv", + "it/25/so.csv", + "it/25/pv.csv", + "it/25/mn.csv", + "it/25/va.csv", + "it/25/lo.csv", + "it/25/mi.csv", + "it/52/firenze.csv", + "ee/countrywide.csv", + "lt/city_of_vilnius.csv", + "dk/countrywide.csv", + "lu/countrywide.csv", + "gr/b/municipality_of_kalamaria.csv", + "cz/countrywide.csv", + "pt/countrywide.csv", + "se/municipality_of_gislaved.csv", + "se/municipality_of_vaxjo.csv", + "se/municipality_of_kalmar.csv", + "se/municipality_of_stockholm.csv", + "se/municipality_of_vaxholm.csv", + "se/municipality_of_alingsas.csv", + "se/municipality_of_helsingborg.csv", + "se/municipality_of_nacka.csv", + "se/municipality_of_malmo.csv", + "se/municipality_of_savsjö.csv", + "se/municipality_of_uppsala.csv", + "se/municipality_of_Österåker.csv", + "se/municipality_of_kristinehamn.csv", + "se/municipality_of_hoganas.csv", + "se/municipality_of_vasteras.csv", + "at/tirol.csv", + "at/31256.csv", + "at/31255.csv", + "at/31254.csv", + "at/city_of_vienna.csv", + "es/25831.csv", + "es/32628.csv", + "es/25829.csv", + "es/25830.csv", + "es/nc/statewide.csv", + "be/bru/bosa-region-brussels-fr.csv", + "be/bru/bosa-region-brussels-nl.csv", + "be/wal/bosa-region-wallonia-fr.csv", + "be/wal/bosa-region-wallonia-de.csv", + "be/vlg/bosa-region-flanders-fr.csv", + "be/vlg/bosa-region-flanders-nl.csv", + "nl/countrywide.csv" + ] + }, + "polyline": { + "datapath": "/data/polylines", + "files": [ "extract.0sv" ] + }, + "whosonfirst": { + "datapath": "/data/whosonfirst", + "importPostalcodes": true, + "importPlace": [ "102191581" ] + } + } +}