Add support for pluggable storage backends to mod_tile / renderd

So far, (meta)tiles could only be stored and retrieved from a locally mounted posix filesystem. Although, the posix filesystem API is it self a plugable storage layer into which one can mount many different filsystems, from temporary filesystems to network filesystems, as mod_tile installations scale up to multi-server environments this might not be sufficient and one may want to use other storage layers. This patch therefore abstracts all storage calls out into a separate API that can be implemented by various storage backends to fit the needs of different installations. Three storage backends are included in this commit: - file backend: This is the equivalent of what existed before in mod_tile / renderd. This uses a posix filesystem to store (meta)tiles - memcached: This stores tiles in a memcached store - rados: This stores tiles in a rados / ceph cluster The memcached and rados backends should currently still be considered as experimental There are also other refactoring and cleanups in this commit
openstreetmap · Mar 23, 2013 · e8001c8 · e8001c8
1 parent 23fb781
commit e8001c8
Show file tree

Hide file tree

Showing 26 changed files with 1,860 additions and 1,454 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -5,26 +5,30 @@ ACLOCAL_AMFLAGS = -I m4
 
 AM_CPPFLAGS = $(FT2_CFLAGS) $(PTHREAD_CFLAGS) $(BOOST_CPPFLAGS) $(ICU_CPPFLAGS)
 
-bin_PROGRAMS = render_old render_expired render_list speedtest render_old renderd convert_meta
+STORE_SOURCES = store.c store_file.c store_file_utils.c store_memcached.c store_rados.c
+STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS)
+
+bin_PROGRAMS = renderd render_expired render_list render_speedtest render_old
 
 renderddir = $(sysconfdir)
-renderd_SOURCES = protocol.h store.c daemon.h daemon.c daemon_compat.c gen_tile.h gen_tile.cpp dir_utils.c sys_utils.c sys_utils.h render_config.h dir_utils.h store.h iniparser3.0b/libiniparser.la
-renderd_LDADD = $(FT2_LIBS) $(PTHREAD_CFLAGS) $(MAPNIK_LDFLAGS) $(BOOST_LDFLAGS) $(ICU_LDFLAGS) -Liniparser3.0b/.libs -liniparser
+renderd_SOURCES = protocol.h daemon.h daemon.c daemon_compat.c gen_tile.h gen_tile.cpp sys_utils.c sys_utils.h render_config.h $(STORE_SOURCES) iniparser3.0b/libiniparser.la 
+renderd_LDADD = $(FT2_LIBS) $(PTHREAD_CFLAGS) $(MAPNIK_LDFLAGS) $(BOOST_LDFLAGS) $(ICU_LDFLAGS) $(STORE_LDFLAGS) -Liniparser3.0b/.libs -liniparser
 renderd_DATA = renderd.conf
-speedtest_SOURCES = render_config.h protocol.h dir_utils.c dir_utils.h speedtest.cpp
-render_list_SOURCES = render_config.h protocol.h dir_utils.c dir_utils.h render_list.c sys_utils.c sys_utils.h
-render_list_LDADD = $(PTHREAD_CFLAGS)
-render_expired_SOURCES = render_config.h protocol.h dir_utils.c dir_utils.h render_expired.c
-render_expired_LDADD = $(PTHREAD_CFLAGS)
-render_old_SOURCES = render_config.h protocol.h dir_utils.c dir_utils.h render_old.c sys_utils.c sys_utils.h
+render_speedtest_SOURCES = render_config.h protocol.h speedtest.cpp render_submit_queue.c sys_utils.c
+render_speedtest_LDADD = $(PTHREAD_CFLAGS)
+render_list_SOURCES = render_config.h protocol.h render_list.c sys_utils.c sys_utils.h render_submit_queue.c $(STORE_SOURCES)
+render_list_LDADD = $(PTHREAD_CFLAGS) $(STORE_LDFLAGS)
+render_expired_SOURCES = render_config.h protocol.h render_expired.c render_submit_queue.c sys_utils.c $(STORE_SOURCES)
+render_expired_LDADD = $(PTHREAD_CFLAGS) $(STORE_LDFLAGS)
+render_old_SOURCES = render_config.h protocol.h store_file_utils.c render_old.c sys_utils.c sys_utils.h render_submit_queue.c
 render_old_LDADD = $(PTHREAD_CFLAGS)
-convert_meta_SOURCES = render_config.h protocol.h dir_utils.c dir_utils.h store.c convert_meta.c
+#convert_meta_SOURCES = render_config.h protocol.h dir_utils.c dir_utils.h store.c convert_meta.c
 
 all-local:
-	$(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) $(AM_LDFLAGS) @srcdir@/mod_tile.c @srcdir@/store.c @srcdir@/dir_utils.c @srcdir@/sys_utils.c
+	$(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/mod_tile.c  @srcdir@/sys_utils.c @srcdir@/store.c @srcdir@/store_file.c @srcdir@/store_file_utils.c @srcdir@/store_memcached.c @srcdir@/store_rados.c
 
 install-mod_tile: 
 	mkdir -p $(DESTDIR)`$(APXS) -q LIBEXECDIR`
-	$(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) $(AM_LDFLAGS) @srcdir@/mod_tile.c @srcdir@/store.c @srcdir@/dir_utils.c @srcdir@/sys_utils.c
+	$(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/mod_tile.c @srcdir@/sys_utils.c @srcdir@/store.c @srcdir@/store_file.c @srcdir@/store_file_utils.c @srcdir@/store_memcached.c @srcdir@/store_rados.c  
 
 
diff --git a/configure.ac b/configure.ac
@@ -41,6 +41,12 @@ AC_SEARCH_LIBS(pow,m)
 AX_BOOST_BASE
 AC_CHECK_ICU(4.2)
 AX_LIB_MAPNIK
+AX_ENABLE_LIBMEMCACHED
+AC_CHECK_LIB(rados, rados_version, [
+    AC_DEFINE([HAVE_LIBRADOS], [1], [Have found librados])
+    LIBRADOS_LDFLAGS='-lrados'
+    AC_SUBST(LIBRADOS_LDFLAGS)
+][])
 AC_CHECK_FUNCS([bzero gethostbyname gettimeofday inet_ntoa memset mkdir pow select socket strchr strdup strerror strrchr strstr strtol strtoul utime],[],[AC_MSG_ERROR([One of the required functions was not found])])
 AC_CHECK_FUNCS([daemon getloadavg],[],[])
 

diff --git a/daemon.c b/daemon.c
@@ -23,7 +23,6 @@
 #include "daemon.h"
 #include "gen_tile.h"
 #include "protocol.h"
-#include "dir_utils.h"
 
 #define PIDFILE "/var/run/renderd/renderd.pid"
 
@@ -301,6 +300,23 @@ enum protoCmd pending(struct item *test)
     return cmdRender;
 }
 
+static int check_xyz(int x, int y, int z)
+{
+    int oob, limit;
+
+    // Validate tile co-ordinates
+    oob = (z < 0 || z > MAX_ZOOM);
+    if (!oob) {
+         // valid x/y for tiles are 0 ... 2^zoom-1
+        limit = (1 << z) - 1;
+        oob =  (x < 0 || x > limit || y < 0 || y > limit);
+    }
+
+    if (oob)
+        fprintf(stderr, "got bad co-ords: x(%d) y(%d) z(%d)\n", x, y, z);
+
+    return oob;
+}
 
 enum protoCmd rx_request(const struct protocol *req, int fd)
 {

diff --git a/gen_tile.cpp b/gen_tile.cpp
@@ -20,9 +20,9 @@
 #include "gen_tile.h"
 #include "render_config.h"
 #include "daemon.h"
-#include "protocol.h"
-#include "dir_utils.h"
 #include "store.h"
+#include "metatile.h"
+#include "protocol.h"
 
 #ifdef HTCP_EXPIRE_CACHE
 #include <sys/socket.h>
@@ -60,7 +60,7 @@ static const int maxZoom = MAX_ZOOM;
 struct xmlmapconfig {
     char xmlname[XMLCONFIG_MAX];
     char xmlfile[PATH_MAX];
-    char tile_dir[PATH_MAX];
+    struct storage_backend * store;
     Map map;
     projection prj;
     char xmluri[PATH_MAX];
@@ -339,88 +339,71 @@ class metaTile {
             return (x & mask) * METATILE + (y & mask);
         }
 
-        void save(const char *tile_dir)
+        void save(struct storage_backend * store)
         {
             int ox, oy, limit;
             size_t offset;
             struct meta_layout m;
-            char meta_path[PATH_MAX];
             struct entry offsets[METATILE * METATILE];
+            char * metatilebuffer;
 
             memset(&m, 0, sizeof(m));
             memset(&offsets, 0, sizeof(offsets));
 
-            xyz_to_meta(meta_path, sizeof(meta_path), tile_dir, xmlconfig_.c_str(), x_, y_, z_);
-            std::stringstream ss;
-            ss << std::string(meta_path) << "." << pthread_self();
-            std::string tmp(ss.str());
-
-            if (mkdirp(tmp.c_str()))
-	      throw std::runtime_error("An error when creating tile directory");
-
-	    try {
-	      std::ofstream file;
-	      file.exceptions(std::ios_base::failbit | std::ios_base::badbit);
-	      file.open(tmp.c_str(), std::ios::out | std::ios::binary | std::ios::trunc);
-
-	      // Create and write header
-	      m.count = METATILE * METATILE;
-	      memcpy(m.magic, META_MAGIC, strlen(META_MAGIC));
-	      m.x = x_;
-	      m.y = y_;
-	      m.z = z_;
-	      file.write((const char *)&m, sizeof(m));
-
-	      offset = header_size;
-	      limit = (1 << z_);
-	      limit = MIN(limit, METATILE);
-
-	      // Generate offset table
-	      for (ox=0; ox < limit; ox++) {
-                for (oy=0; oy < limit; oy++) {
-		  int mt = xyz_to_meta_offset(x_ + ox, y_ + oy, z_);
-		  offsets[mt].offset = offset;
-		  offsets[mt].size   = tile[ox][oy].size();
-		  offset += offsets[mt].size;
-                }
-	      }
-	      file.write((const char *)&offsets, sizeof(offsets));
-
-	      // Write tiles
-	      for (ox=0; ox < limit; ox++) {
+            // Create and write header
+            m.count = METATILE * METATILE;
+            memcpy(m.magic, META_MAGIC, strlen(META_MAGIC));
+            m.x = x_;
+            m.y = y_;
+            m.z = z_;
+
+            offset = header_size;
+            limit = (1 << z_);
+            limit = MIN(limit, METATILE);
+
+            // Generate offset table
+            for (ox=0; ox < limit; ox++) {
                 for (oy=0; oy < limit; oy++) {
-		  file.write((const char *)tile[ox][oy].data(), tile[ox][oy].size());
+                    int mt = xyz_to_meta_offset(x_ + ox, y_ + oy, z_);
+                    offsets[mt].offset = offset;
+                    offsets[mt].size   = tile[ox][oy].size();
+                    offset += offsets[mt].size;
                 }
-	      }
-
-	      file.close();
-
-	      rename(tmp.c_str(), meta_path);
-	    } catch (std::ios_base::failure) {
-	      // Remove the temporary file
-	      unlink(tmp.c_str());
-	      throw std::runtime_error("An error occured when writing a metatile\n");
-	    }
-            //printf("Produced .meta: %s\n", meta_path);
-        }
-
-#ifdef HTCP_EXPIRE_CACHE
-        void expire_tiles(int sock, char * host, char * uri) {
-            if (sock < 0) {
-                return;
             }
-            syslog(LOG_INFO, "Purging metatile via HTCP cache expiry");
-            int ox, oy;
-            int limit = (1 << z_);
-            limit = MIN(limit, METATILE);
 
-            // Generate offset table
+            metatilebuffer = (char *) malloc(offset);
+            memset(metatilebuffer, 0, offset);
+            memcpy(metatilebuffer,&m,sizeof(m));
+            memcpy(metatilebuffer + sizeof(m), &offsets, sizeof(offsets));
+
+            // Write tiles
             for (ox=0; ox < limit; ox++) {
                 for (oy=0; oy < limit; oy++) {
-                    cache_expire(sock, host, uri, (x_ + ox), (y_ + oy), z_);
+                    memcpy(metatilebuffer + offsets[xyz_to_meta_offset(x_ + ox, y_ + oy, z_)].offset, (const void *)tile[ox][oy].data(), tile[ox][oy].size());
                 }
             }
+
+            store->metatile_write(store, xmlconfig_.c_str(),x_,y_,z_,metatilebuffer, offset);
+
+        }
+
+#ifdef HTCP_EXPIRE_CACHE
+    void expire_tiles(int sock, char * host, char * uri) {
+        if (sock < 0) {
+            return;
+        }
+        syslog(LOG_INFO, "Purging metatile via HTCP cache expiry");
+        int ox, oy;
+        int limit = (1 << z_);
+        limit = MIN(limit, METATILE);
+
+        // Generate offset table
+        for (ox=0; ox < limit; ox++) {
+            for (oy=0; oy < limit; oy++) {
+                cache_expire(sock, host, uri, (x_ + ox), (y_ + oy), z_);
+            }
         }
+    }
 #endif
         int x_, y_, z_;
         std::string xmlconfig_;
@@ -473,7 +456,7 @@ static enum protoCmd render(Map &m, char *xmlname, projection &prj, int x, int y
 //    syslog(LOG_DEBUG, "DEBUG: DONE TILE %s %d %d-%d %d-%d", xmlname, z, x, x+size-1, y, y+size-1);
     return cmdDone; // OK
 }
-#else
+#else //METATILE
 static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, projection &prj, int x, int y, int z)
 {
     char filename[PATH_MAX];
@@ -513,7 +496,7 @@ static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, project
 
     return cmdDone; // OK
 }
-#endif
+#endif //METATILE
 
 
 void render_init(const char *plugins_dir, const char* font_dir, int font_dir_recurse)
@@ -537,35 +520,42 @@ void *render_thread(void * arg)
         if (parentxmlconfig[iMaxConfigs].xmlname[0] == 0 || parentxmlconfig[iMaxConfigs].xmlfile[0] == 0) break;
         strcpy(maps[iMaxConfigs].xmlname, parentxmlconfig[iMaxConfigs].xmlname);
         strcpy(maps[iMaxConfigs].xmlfile, parentxmlconfig[iMaxConfigs].xmlfile);
-        strcpy(maps[iMaxConfigs].tile_dir, parentxmlconfig[iMaxConfigs].tile_dir);
-        maps[iMaxConfigs].map = mapnik::Map(RENDER_SIZE, RENDER_SIZE);
-        maps[iMaxConfigs].ok = 1;
-	try {
-	  mapnik::load_map(maps[iMaxConfigs].map, maps[iMaxConfigs].xmlfile);
-	} catch (std::exception const& ex) {
-	  syslog(LOG_ERR, "An error occurred while loading the map layer '%s': %s", maps[iMaxConfigs].xmlname, ex.what());
-	  maps[iMaxConfigs].ok = 0;
-	} catch (...) {
-	  syslog(LOG_ERR, "An unknown error occurred while loading the map layer '%s'", maps[iMaxConfigs].xmlname);
-	  maps[iMaxConfigs].ok = 0;
-	}
-        maps[iMaxConfigs].prj = projection(maps[iMaxConfigs].map.srs());
+        maps[iMaxConfigs].store = init_storage_backend(parentxmlconfig[iMaxConfigs].tile_dir);
+
+        if (maps[iMaxConfigs].store) {
+            maps[iMaxConfigs].ok = 1;
+
+            maps[iMaxConfigs].map = mapnik::Map(RENDER_SIZE, RENDER_SIZE);
+
+            try {
+                mapnik::load_map(maps[iMaxConfigs].map, maps[iMaxConfigs].xmlfile);
+            } catch (std::exception const& ex) {
+                syslog(LOG_ERR, "An error occurred while loading the map layer '%s': %s", maps[iMaxConfigs].xmlname, ex.what());
+                maps[iMaxConfigs].ok = 0;
+            } catch (...) {
+                syslog(LOG_ERR, "An unknown error occurred while loading the map layer '%s'", maps[iMaxConfigs].xmlname);
+                maps[iMaxConfigs].ok = 0;
+            }
+            maps[iMaxConfigs].prj = projection(maps[iMaxConfigs].map.srs());
 #ifdef HTCP_EXPIRE_CACHE
-        strcpy(maps[iMaxConfigs].xmluri, parentxmlconfig[iMaxConfigs].xmluri);
-        strcpy(maps[iMaxConfigs].host, parentxmlconfig[iMaxConfigs].host);
-        strcpy(maps[iMaxConfigs].htcphost, parentxmlconfig[iMaxConfigs].htcpip);
-        if (strlen(maps[iMaxConfigs].htcphost) > 0) {
-            maps[iMaxConfigs].htcpsock = init_cache_expire(
-                    maps[iMaxConfigs].htcphost);
-            if (maps[iMaxConfigs].htcpsock > 0) {
-                syslog(LOG_INFO, "Successfully opened socket for HTCP cache expiry");
+            strcpy(maps[iMaxConfigs].xmluri, parentxmlconfig[iMaxConfigs].xmluri);
+            strcpy(maps[iMaxConfigs].host, parentxmlconfig[iMaxConfigs].host);
+            strcpy(maps[iMaxConfigs].htcphost, parentxmlconfig[iMaxConfigs].htcpip);
+            if (strlen(maps[iMaxConfigs].htcphost) > 0) {
+                maps[iMaxConfigs].htcpsock = init_cache_expire(
+                        maps[iMaxConfigs].htcphost);
+                if (maps[iMaxConfigs].htcpsock > 0) {
+                    syslog(LOG_INFO, "Successfully opened socket for HTCP cache expiry");
+                } else {
+                    syslog(LOG_ERR, "Failed to opened socket for HTCP cache expiry");
+                }
             } else {
-                syslog(LOG_ERR, "Failed to opened socket for HTCP cache expiry");
+                maps[iMaxConfigs].htcpsock = -1;
             }
-        } else {
-            maps[iMaxConfigs].htcpsock = -1;
-        }
+
 #endif
+        } else
+            maps[iMaxConfigs].ok = 0;
     }
 
     while (1) {
@@ -599,7 +589,7 @@ void *render_thread(void * arg)
 
                     if (ret == cmdDone) {
                         try {
-                            tiles.save(maps[i].tile_dir);
+                            tiles.save(maps[i].store);
                         } catch (...) {
                             // Treat any error as fatal and request end of processing
                             syslog(LOG_ERR, "Received error when writing metatile to disk, requesting exit.");