vectortile: Continue setup of vector tile servers

This fixes some issues and creates a working server, without updates.

cookbooks/postgresql/libraries/postgresql.rb

@@ -133,7 +133,7 @@ def schemas(database)
 
     def tables(database)
       @tables ||= {}
-      @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind = 'r'", :database => database).each_with_object({}) do |table, tables|
+      @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind IN ('r', 'p')", :database => database).each_with_object({}) do |table, tables|
         name = "#{table[:nspname]}.#{table[:relname]}"
 
         tables[name] = {

cookbooks/tile/attributes/default.rb

@@ -1,6 +1,6 @@
 default[:tile][:database][:cluster] = "14/main"
 default[:tile][:database][:postgis] = "3"
-default[:tile][:database][:node_file] = "/store/database/nodes"
+default[:tile][:database][:nodes_store] = :flat
 default[:tile][:database][:multi_geometry] = true
 default[:tile][:database][:hstore] = true
 default[:tile][:database][:style_file] = nil

cookbooks/vectortile/attributes/default.rb

@@ -1,6 +1,7 @@
 default[:vectortile][:database][:cluster] = "16/main"
 default[:vectortile][:database][:postgis] = "3"
 default[:vectortile][:serve][:threads] = node.cpu_cores
+default[:vectortile][:serve][:mode] = :live
 default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute"
 
 default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first]

cookbooks/vectortile/recipes/default.rb

@@ -83,6 +83,8 @@
   group "tileupdate"
 end
 
+shortbread_config = "#{style_directory}/shortbread.yaml"
+
 themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark"
 git themepark_directory do
   repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git"
@@ -115,19 +117,21 @@
   mode "755"
 end
 
+node_store_options = node[:vectortile][:database][:nodes_store] == :flat ? "--flat-nodes '/srv/vector.openstreetmap.org/data/nodes.bin'" : ""
 template "/usr/local/bin/import-planet" do
   source "import-planet.erb"
   owner "root"
   group "root"
   mode "755"
+  variables :node_store_options => "#{node_store_options}"
 end
 
 template "/usr/local/bin/tilekiln-storage-init" do
   source "tilekiln-storage-init.erb"
   owner "root"
   group "root"
   mode "755"
-  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{style_directory}/shortbread.yaml"
+  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{shortbread_config}"
 end
 
 postgresql_user "tomh" do
@@ -177,24 +181,42 @@
   user "tileupdate"
 end
 
-postgresql_table "metadata" do
-  cluster node[:vectortile][:database][:cluster]
-  database "tiles"
-  schema "tilekiln"
-  owner "tileupdate"
-  permissions "tileupdate" => :all, "tilekiln" => :select
+%w[metadata shortbread_v1].each do |table|
+  postgresql_table table do
+      cluster node[:vectortile][:database][:cluster]
+      database "tiles"
+      schema "tilekiln"
+      owner "tileupdate"
+      permissions "tileupdate" => :all, "tilekiln" => :select
+  end
 end
 
+tilekiln_mode = node[:vectortile][:serve][:mode] == :live ? "live --config #{shortbread_config} --source-dbname spirit" : "static"
+
 systemd_service "tilekiln" do
   description "Tilekiln vector tile server"
   user "tilekiln"
   after "postgresql.service"
   wants "postgresql.service"
   sandbox :enable_network => true
   restrict_address_families "AF_UNIX"
-  exec_start "#{tilekiln_directory}/bin/tilekiln serve static --storage-dbname tiles --num-threads #{node[:vectortile][:serve][:threads]}"
+  exec_start "#{tilekiln_directory}/bin/tilekiln serve #{tilekiln_mode} --storage-dbname tiles --num-threads #{node[:vectortile][:serve][:threads]}"
 end
 
 service "tilekiln" do
   action [:enable, :start]
 end
+
+package %w[
+  gdal-bin
+  python3-yaml
+  python3-psycopg2
+]
+
+execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do
+  command "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py -R tilekiln"
+  cwd "/srv/vector.openstreetmap.org/spirit"
+  user "tileupdate"
+  group "tileupdate"
+  ignore_failure true
+end

cookbooks/vectortile/templates/default/import-planet.erb

@@ -1,8 +1,8 @@
 #!/bin/sh
 
 # Usage
-# sudo -u tileupdate import osmfile.osm.pbf
-# sudo -u tileupdate import --cache 50000 osmfile.osm.pbf
+# sudo -u tileupdate import-planet osmfile.osm.pbf
+# sudo -u tileupdate import-planet --cache 50000 osmfile.osm.pbf
 # This script sets the appropriate Lua and style paths for the osm2pgsql import
 
 set -e
@@ -12,13 +12,13 @@ export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/sr
 # Import the osm2pgsql file specified as an argument, using the locations for spirit
 osm2pgsql \
   --output flex \
-  --style '/srv/vector.openstreetmap.org/spirit/spirit.lua' \
+  --style '/srv/vector.openstreetmap.org/spirit/shortbread.lua' \
   --slim \
-  --flat-nodes '/srv/vector.openstreetmap.org/data/nodes.bin' \
+  <%= @node_store_options %> \
   -d spirit \
   $@
 
 # Set up replication. This doesn't specify the replication server, so it will use planet.osm.org on extracts
 osm2pgsql-replication init \
   -d spirit \
-  --server '<%= node[:vectortile][:database][:tag_transform_script] %>'
+  --server '<%= node[:vectortile][:replication][:url] %>'

cookbooks/vectortile/templates/default/nginx.erb

@@ -1,3 +1,7 @@
+upstream tilekiln {
+    server 127.0.0.1:8000;
+}
+
 server {
     listen 80 default_server;
     listen [::]:80 default_server;
@@ -35,6 +39,8 @@ server {
         allow ::1;
         deny all;
     }
-
+    location /shortbread_v1/ {
+        proxy_pass http://tilekiln;
+    }
     root /srv/vector.openstreetmap.org/html;
 }

test/integration/vectortile/inspec/tilekiln_spec.rb

@@ -16,3 +16,15 @@
 describe http("http://localhost:8000") do
   its("status") { should cmp 404 }
 end
+
+describe http("https://localhost/shortbread_v1/tilejson.json", :ssl_verify => false) do
+  its("status") { should cmp 200 }
+end
+
+# There are no tiles so everything should return a 404
+describe http("https://localhost/shortbread_v1/0/0/0.mvt", :ssl_verify => false) do
+  its("status") { should cmp 404 }
+end
+describe http("https://localhost/shortbread_v1/16/0/0.mvt", :ssl_verify => false) do
+  its("status") { should cmp 404 }
+end