From c265a370a22156bdc9993587b48c4dd7bcb7c384 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Thu, 29 Aug 2024 17:30:43 -0700 Subject: [PATCH] vectortile: Continue setup of vector tile servers This fixes some issues and creates a working server, without updates. --- cookbooks/postgresql/libraries/postgresql.rb | 2 +- cookbooks/tile/attributes/default.rb | 2 +- cookbooks/vectortile/attributes/default.rb | 1 + cookbooks/vectortile/recipes/default.rb | 38 +++++++++++++++---- .../templates/default/import-planet.erb | 10 ++--- .../vectortile/templates/default/nginx.erb | 8 +++- .../vectortile/inspec/tilekiln_spec.rb | 12 ++++++ 7 files changed, 57 insertions(+), 16 deletions(-) diff --git a/cookbooks/postgresql/libraries/postgresql.rb b/cookbooks/postgresql/libraries/postgresql.rb index dacf885db..b2df4aed6 100644 --- a/cookbooks/postgresql/libraries/postgresql.rb +++ b/cookbooks/postgresql/libraries/postgresql.rb @@ -133,7 +133,7 @@ def schemas(database) def tables(database) @tables ||= {} - @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind = 'r'", :database => database).each_with_object({}) do |table, tables| + @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind IN ('r', 'p')", :database => database).each_with_object({}) do |table, tables| name = "#{table[:nspname]}.#{table[:relname]}" tables[name] = { diff --git a/cookbooks/tile/attributes/default.rb b/cookbooks/tile/attributes/default.rb index cc5803d00..2308d63b3 100644 --- a/cookbooks/tile/attributes/default.rb +++ b/cookbooks/tile/attributes/default.rb @@ -1,6 +1,6 @@ default[:tile][:database][:cluster] = "14/main" default[:tile][:database][:postgis] = "3" -default[:tile][:database][:node_file] = "/store/database/nodes" +default[:tile][:database][:nodes_store] = :flat default[:tile][:database][:multi_geometry] = true default[:tile][:database][:hstore] = true default[:tile][:database][:style_file] = nil diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb index e00130777..0809544ac 100644 --- a/cookbooks/vectortile/attributes/default.rb +++ b/cookbooks/vectortile/attributes/default.rb @@ -1,6 +1,7 @@ default[:vectortile][:database][:cluster] = "16/main" default[:vectortile][:database][:postgis] = "3" default[:vectortile][:serve][:threads] = node.cpu_cores +default[:vectortile][:serve][:mode] = :live default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute" default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first] diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb index 03cc6c259..f4501d5d5 100644 --- a/cookbooks/vectortile/recipes/default.rb +++ b/cookbooks/vectortile/recipes/default.rb @@ -83,6 +83,8 @@ group "tileupdate" end +shortbread_config = "#{style_directory}/shortbread.yaml" + themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark" git themepark_directory do repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git" @@ -115,11 +117,13 @@ mode "755" end +node_store_options = node[:vectortile][:database][:nodes_store] == :flat ? "--flat-nodes '/srv/vector.openstreetmap.org/data/nodes.bin'" : "" template "/usr/local/bin/import-planet" do source "import-planet.erb" owner "root" group "root" mode "755" + variables :node_store_options => "#{node_store_options}" end template "/usr/local/bin/tilekiln-storage-init" do @@ -127,7 +131,7 @@ owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{style_directory}/shortbread.yaml" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{shortbread_config}" end postgresql_user "tomh" do @@ -177,14 +181,18 @@ user "tileupdate" end -postgresql_table "metadata" do - cluster node[:vectortile][:database][:cluster] - database "tiles" - schema "tilekiln" - owner "tileupdate" - permissions "tileupdate" => :all, "tilekiln" => :select +%w[metadata shortbread_v1].each do |table| + postgresql_table table do + cluster node[:vectortile][:database][:cluster] + database "tiles" + schema "tilekiln" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => :select + end end +tilekiln_mode = node[:vectortile][:serve][:mode] == :live ? "live --config #{shortbread_config} --source-dbname spirit" : "static" + systemd_service "tilekiln" do description "Tilekiln vector tile server" user "tilekiln" @@ -192,9 +200,23 @@ wants "postgresql.service" sandbox :enable_network => true restrict_address_families "AF_UNIX" - exec_start "#{tilekiln_directory}/bin/tilekiln serve static --storage-dbname tiles --num-threads #{node[:vectortile][:serve][:threads]}" + exec_start "#{tilekiln_directory}/bin/tilekiln serve #{tilekiln_mode} --storage-dbname tiles --num-threads #{node[:vectortile][:serve][:threads]}" end service "tilekiln" do action [:enable, :start] end + +package %w[ + gdal-bin + python3-yaml + python3-psycopg2 +] + +execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do + command "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py -R tilekiln" + cwd "/srv/vector.openstreetmap.org/spirit" + user "tileupdate" + group "tileupdate" + ignore_failure true +end diff --git a/cookbooks/vectortile/templates/default/import-planet.erb b/cookbooks/vectortile/templates/default/import-planet.erb index 13791972c..bffa34a3a 100644 --- a/cookbooks/vectortile/templates/default/import-planet.erb +++ b/cookbooks/vectortile/templates/default/import-planet.erb @@ -1,8 +1,8 @@ #!/bin/sh # Usage -# sudo -u tileupdate import osmfile.osm.pbf -# sudo -u tileupdate import --cache 50000 osmfile.osm.pbf +# sudo -u tileupdate import-planet osmfile.osm.pbf +# sudo -u tileupdate import-planet --cache 50000 osmfile.osm.pbf # This script sets the appropriate Lua and style paths for the osm2pgsql import set -e @@ -12,13 +12,13 @@ export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/sr # Import the osm2pgsql file specified as an argument, using the locations for spirit osm2pgsql \ --output flex \ - --style '/srv/vector.openstreetmap.org/spirit/spirit.lua' \ + --style '/srv/vector.openstreetmap.org/spirit/shortbread.lua' \ --slim \ - --flat-nodes '/srv/vector.openstreetmap.org/data/nodes.bin' \ + <%= @node_store_options %> \ -d spirit \ $@ # Set up replication. This doesn't specify the replication server, so it will use planet.osm.org on extracts osm2pgsql-replication init \ -d spirit \ - --server '<%= node[:vectortile][:database][:tag_transform_script] %>' + --server '<%= node[:vectortile][:replication][:url] %>' diff --git a/cookbooks/vectortile/templates/default/nginx.erb b/cookbooks/vectortile/templates/default/nginx.erb index 866dfc2be..838d30a4f 100644 --- a/cookbooks/vectortile/templates/default/nginx.erb +++ b/cookbooks/vectortile/templates/default/nginx.erb @@ -1,3 +1,7 @@ +upstream tilekiln { + server 127.0.0.1:8000; +} + server { listen 80 default_server; listen [::]:80 default_server; @@ -35,6 +39,8 @@ server { allow ::1; deny all; } - + location /shortbread_v1/ { + proxy_pass http://tilekiln; + } root /srv/vector.openstreetmap.org/html; } diff --git a/test/integration/vectortile/inspec/tilekiln_spec.rb b/test/integration/vectortile/inspec/tilekiln_spec.rb index 49ae43e91..2887571db 100644 --- a/test/integration/vectortile/inspec/tilekiln_spec.rb +++ b/test/integration/vectortile/inspec/tilekiln_spec.rb @@ -16,3 +16,15 @@ describe http("http://localhost:8000") do its("status") { should cmp 404 } end + +describe http("https://localhost/shortbread_v1/tilejson.json", :ssl_verify => false) do + its("status") { should cmp 200 } +end + +# There are no tiles so everything should return a 404 +describe http("https://localhost/shortbread_v1/0/0/0.mvt", :ssl_verify => false) do + its("status") { should cmp 404 } +end +describe http("https://localhost/shortbread_v1/16/0/0.mvt", :ssl_verify => false) do + its("status") { should cmp 404 } +end