Merge remote-tracking branch 'github/pull/686'

.kitchen.yml

@@ -416,6 +416,9 @@ suites:
   - name: trac
     run_list:
       - recipe[trac::default]
+  - name: vectortile
+    run_list:
+      - recipe[vectortile::default]
   - name: web-cgimap
     run_list:
       - recipe[web::cgimap]

cookbooks/postgresql/libraries/postgresql.rb

@@ -4,6 +4,10 @@ module OpenStreetMap
   class PostgreSQL
     include Chef::Mixin::ShellOut
 
+    SCHEMA_PRIVILEGES = [
+      :create, :usage
+    ].freeze
+
     TABLE_PRIVILEGES = [
       :select, :insert, :update, :delete, :truncate, :references, :trigger
     ].freeze
@@ -115,9 +119,21 @@ def tablespaces
       end
     end
 
+    def schemas(database)
+      @schemas ||= {}
+      @schemas[database] ||= query("SELECT n.nspname, pg_catalog.pg_get_userbyid(n.nspowner) AS usename, n.nspacl FROM pg_namespace AS n WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema'", :database => database).each_with_object({}) do |schema, schemas|
+        name = "#{schema[:nspname]}"
+
+        schemas[name] = {
+          :owner => schema[:usename],
+          :permissions => parse_acl(schema[:nspacl] || "{}")
+        }
+      end
+    end
+
     def tables(database)
       @tables ||= {}
-      @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind = 'r'", :database => database).each_with_object({}) do |table, tables|
+      @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind IN ('r', 'p')", :database => database).each_with_object({}) do |table, tables|
         name = "#{table[:nspname]}.#{table[:relname]}"
 
         tables[name] = {

cookbooks/postgresql/resources/schema.rb

@@ -0,0 +1,114 @@
+#
+# Cookbook:: postgresql
+# Resource:: postgresql_schema
+#
+# Copyright:: 2024, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+unified_mode true
+
+default_action :create
+
+property :schema, :kind_of => String, :name_property => true
+property :cluster, :kind_of => String, :required => true
+property :database, :kind_of => String, :required => true
+property :owner, :kind_of => String, :required => [:create]
+property :permissions, :kind_of => Hash, :default => {}
+
+action :create do
+  if schemas.include?(qualified_name)
+    # Handle the case of an existing schema
+    if new_resource.owner != schemas[qualified_name][:owner]
+      converge_by("set owner for #{new_resource} to #{new_resource.owner}") do
+        Chef::Log.info("Setting owner for #{new_resource} to #{new_resource.owner}")
+        cluster.execute(:command => "ALTER SCHEMA #{qualified_name} OWNER TO \"#{new_resource.owner}\"", :database => new_resource.database)
+      end
+    end
+
+    schemas[qualified_name][:permissions].each_key do |user|
+      next if new_resource.permissions[user]
+      # Remove permissions from users who no longer have them
+      converge_by("revoke all for #{user} on #{new_resource}") do
+        Chef::Log.info("Revoking all for #{user} on #{new_resource}")
+        cluster.execute(:command => "REVOKE ALL ON SCHEMA #{qualified_name} FROM \"#{user}\"", :database => new_resource.database)
+      end
+    end
+    new_resource.permissions.each do |user, new_privileges|
+      current_privileges = schemas[qualified_name][:permissions][user] || {}
+      new_privileges = Array(new_privileges)
+
+      if new_privileges.include?(:all)
+        new_privileges |= OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES
+      end
+
+      OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES.each do |privilege|
+        if new_privileges.include?(privilege)
+          unless current_privileges.include?(privilege)
+            converge_by("grant #{privilege} for #{user} on #{new_resource}") do
+              Chef::Log.info("Granting #{privilege} for #{user} on #{new_resource}")
+              cluster.execute(:command => "GRANT #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} TO \"#{user}\"", :database => new_resource.database)
+            end
+          end
+        elsif current_privileges.include?(privilege)
+          converge_by("revoke #{privilege} for #{user} on #{new_resource}") do
+            Chef::Log.info("Revoking #{privilege} for #{user} on #{new_resource}")
+            cluster.execute(:command => "REVOKE #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} FROM \"#{user}\"", :database => new_resource.database)
+          end
+        end
+      end
+    end
+  else
+    converge_by "create schema #{new_resource.schema}" do
+      cluster.execute(:command => "CREATE SCHEMA #{new_resource.schema} AUTHORIZATION #{new_resource.owner}", :database => new_resource.database)
+      # Because the schema is new, we don't have to worry about revoking or checking current permissions
+      new_resource.permissions.each do |user, new_privileges|
+        new_privileges = Array(new_privileges)
+        if new_privileges.include?(:all)
+          new_privileges |= OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES
+        end
+        OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES.each do |privilege|
+          next unless new_privileges.include?(privilege)
+          converge_by("grant #{privilege} for #{user} on #{new_resource}") do
+            Chef::Log.info("Granting #{privilege} for #{user} on #{new_resource}")
+            cluster.execute(:command => "GRANT #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} TO \"#{user}\"", :database => new_resource.database)
+          end
+        end
+      end
+    end
+  end
+end
+
+action :drop do
+  if schemas.include?(qualified_name)
+    converge_by("drop #{new_resource}") do
+      Chef::Log.info("Dropping #{new_resource}")
+      cluster.execute(:command => "DROP SCHEMA #{qualified_name}", :database => new_resource.database)
+    end
+  end
+end
+
+action_class do
+  def cluster
+    @cluster ||= OpenStreetMap::PostgreSQL.new(new_resource.cluster)
+  end
+
+  def schemas
+    @schemas ||= cluster.schemas(new_resource.database)
+  end
+
+  def qualified_name
+    "#{new_resource.name}"
+  end
+end

cookbooks/vectortile/README.md

@@ -0,0 +1,9 @@
+# vectortile cookbook
+
+This cookbook installs and configures the tilekiln based tileservers
+
+## Accounts
+The following accounts are used
+- `www-data` for nginx serving static files and proxying
+- `tilekiln` for the process serving tiles
+- `tileupdate` for the process running osm2pgsql and tilekiln on updates

cookbooks/vectortile/attributes/default.rb

@@ -0,0 +1,15 @@
+default[:vectortile][:database][:cluster] = "16/main"
+default[:vectortile][:database][:postgis] = "3"
+default[:vectortile][:database][:nodes_store] = :flat
+default[:vectortile][:serve][:threads] = node.cpu_cores
+default[:vectortile][:serve][:mode] = :live
+default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute"
+default[:vectortile][:replication][:status] = :enabled
+default[:vectortile][:replication][:tileupdate] = :enabled
+default[:vectortile][:replication][:threads] = node.cpu_cores
+
+default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first]
+default[:postgresql][:monitor_database] = "tiles"
+default[:postgresql][:settings][:defaults][:max_connections] = (node.cpu_cores * 4 + 20).to_s
+default[:accounts][:users][:tileupdate][:status] = :role
+default[:accounts][:users][:tilekiln][:status] = :role

cookbooks/vectortile/files/default/html/clientaccesspolicy.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<access-policy>
+  <cross-domain-access>
+    <policy>
+      <allow-from http-request-headers="*">
+        <domain uri="*"/>
+      </allow-from>
+      <grant-to>
+        <resource path="/" include-subpaths="true"/>
+      </grant-to>
+    </policy>
+  </cross-domain-access>
+</access-policy>

cookbooks/vectortile/files/default/html/crossdomain.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0"?>
+<!DOCTYPE cross-domain-policy SYSTEM "http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd">
+<cross-domain-policy>
+    <allow-access-from domain="*" secure="false" />
+</cross-domain-policy>

cookbooks/vectortile/files/default/html/robots.txt

@@ -0,0 +1,13 @@
+User-agent: *
+Disallow: /12/
+Disallow: /13/
+Disallow: /14/
+Disallow: /15/
+Disallow: /16/
+Disallow: /17/
+Disallow: /18/
+Disallow: /19/
+Disallow: /20/
+Disallow: /21/
+Disallow: /22/
+Disallow: /23/

cookbooks/vectortile/metadata.rb

@@ -0,0 +1,16 @@
+name              "vectortile"
+maintainer        "OpenStreetMap Administrators"
+maintainer_email  "[email protected]"
+license           "Apache-2.0"
+description       "Installs and configures vector tile servers"
+
+version           "1.0.0"
+supports          "ubuntu"
+depends           "accounts"
+depends           "git"
+depends           "nginx"
+depends           "postgresql"
+depends           "prometheus"
+depends           "python"
+depends           "systemd"
+depends           "tools"