From a2763edfd19d9f7243fb5e3a06b0c417eaa0748a Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 4 Jun 2018 14:48:43 -0700 Subject: [PATCH 01/85] Fixed gzipped tile content handling The gzipped content gets sent as such if the request headers declare support for gzip Otherwise, mod_deflate is used to inflate the content. If mod_deflate is not found when needed, a warning is issued Visual Studio project updated to VS2017 --- mod_mrf.vcxproj | 9 +++++---- src/mod_mrf.cpp | 21 +++++++++++++++++++-- src/mod_mrf.h | 2 +- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index f9431c3..bc17bb1 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -24,20 +24,21 @@ {C7E01836-333F-490D-A3B5-3554A8935040} Win32Proj mod_mrf + 10.0.17134.0 DynamicLibrary true - v120 + v141 NotSet Application false - v120 + v141 true - Unicode + NotSet diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 1b8c731..f014a22 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -358,11 +358,28 @@ static int send_image(request_rec *r, apr_uint32_t *buffer, apr_size_t size) default: // LERC goes here too ap_set_content_type(r, "application/octet-stream"); } + // Is it gzipped content? - if (GZIP_SIG == hton32(*buffer)) + if (GZIP_SIG == hton32(*buffer)) { apr_table_setn(r->headers_out, "Content-Encoding", "gzip"); + const char *ae = apr_table_get(r->headers_in, "Accept-Encoding"); + // If accept encoding is missing, assume it doesn't support gzip + if (!ae || strstr(ae, "gzip")) { + ap_filter_rec_t *inflate_filter = ap_get_output_filter_handle("INFLATE"); + if (inflate_filter) + ap_add_output_filter_handle(inflate_filter, NULL, r, r->connection); + else + ap_log_rerror(APLOG_MARK, APLOG_WARNING, 0, r, + "mod_deflate not loaded, sending gzipped tile to client that does not declare support"); + } + } + + // + // Static headers can be added with the header module + // apr_table_setn(r->headers_out, "Access-Control-Allow-Origin", "*"); + // apt_table_setn(r->headers_out, "Cache-Control", "max-age=86400"); + // - // TODO: Set headers, as chosen by user ap_set_content_length(r, size); ap_rwrite(buffer, size, r); return OK; diff --git a/src/mod_mrf.h b/src/mod_mrf.h index 6b271b5..9defe83 100644 --- a/src/mod_mrf.h +++ b/src/mod_mrf.h @@ -31,7 +31,7 @@ #define LERC_SIG 0x436e745a // This one is not a type, just an encoding -#define GZIP_SIG 0x436e745a +#define GZIP_SIG 0x1f8b0800 // Conversion to and from network order, endianess depenent From 2ce09882c6b6df47e45ed1ca4ec4f965c191d82c Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 4 Jun 2018 14:51:59 -0700 Subject: [PATCH 02/85] Correct ungzip condition --- src/mod_mrf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index f014a22..8797533 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -364,7 +364,7 @@ static int send_image(request_rec *r, apr_uint32_t *buffer, apr_size_t size) apr_table_setn(r->headers_out, "Content-Encoding", "gzip"); const char *ae = apr_table_get(r->headers_in, "Accept-Encoding"); // If accept encoding is missing, assume it doesn't support gzip - if (!ae || strstr(ae, "gzip")) { + if (!ae || !strstr(ae, "gzip")) { ap_filter_rec_t *inflate_filter = ap_get_output_filter_handle("INFLATE"); if (inflate_filter) ap_add_output_filter_handle(inflate_filter, NULL, r, r->connection); From bda95298c14b4930c901f57882ef7cea833c847d Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 1 Aug 2018 17:48:04 -0700 Subject: [PATCH 03/85] Fix for MRFs with index files larger than 2GB c->size.z is 64bit, which will promote the math to 64bit, signed. This is safe until the index file is under 2^63 - 1 bytes. --- src/mod_mrf.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 8797533..fd0c5a9 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -136,7 +136,10 @@ static void mrf_init(apr_pool_t *p, mrf_conf *c) { for (int i = 0; i < c->n_levels; i++) { *r-- = level; // Prepare for the next level, assuming powers of two - level.offset += sizeof(TIdx) * level.width * level.height * c->size.z; + // This is safe on all platforms that have large files (64bit signed offset) + // It will start failing if the file offset is larger than 63bits + // The c->size.z has to be first, to force the 64bit type + level.offset += c->size.z * level.width * level.height * sizeof(TIdx); level.width = 1 + (level.width - 1) / 2; level.height = 1 + (level.height - 1) / 2; } From fe8aa10653c757c12e2ca5a4c18cb0a66a671a26 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 1 Aug 2018 17:49:27 -0700 Subject: [PATCH 04/85] Upgrade to libapr-2 libpar-2 is still in dev, but it seems to work fine --- mod_mrf.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index bc17bb1..9eb01e8 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -71,7 +71,7 @@ Windows true \Apache24\lib - libhttpd.lib;libapr-1.lib;libaprutil-1.lib;%(AdditionalDependencies) + libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) /EXPORT:mrf_module,@1 $(OutDir)$(TargetName)$(TargetExt) From 00cc7b7faf61c95ae848b5aee4fedfec4aa60836 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sat, 18 Aug 2018 17:30:41 -0700 Subject: [PATCH 05/85] Fix subrequest use and memmory leak - Use the subrequest filter stack for receive instead of the main request. This allows stacking of filters - Clean up the subrequest when done --- src/mod_mrf.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index fd0c5a9..cd36fc7 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -590,15 +590,17 @@ static int handler(request_rec *r) rctx.maxsize = static_cast(index.size); rctx.size = 0; - ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, r, r->connection); request_rec *sr = ap_sub_req_lookup_uri(cfg->redirect, r, r->output_filters); // Data file is on a remote site a range request redirect with a range header static const char *rfmt = "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT; char *Range = apr_psprintf(r->pool, rfmt, index.offset, index.offset + index.size); apr_table_setn(sr->headers_in, "Range", Range); + + ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, sr, sr->connection); int status = ap_run_sub_req(sr); ap_remove_output_filter(rf); + ap_destroy_sub_req(sr); if (status != APR_SUCCESS || sr->status != HTTP_PARTIAL_CONTENT || rctx.size != static_cast(index.size)) { From 6bd67551122b1d90221b2463351154bca5ba0788 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sat, 18 Aug 2018 17:31:03 -0700 Subject: [PATCH 06/85] VS project cleanup --- .gitignore | 1 + mod_mrf.vcxproj | 5 +++++ mod_mrf.vcxproj.filters | 25 ++++++++++++++++++++----- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index da346c2..bbab9e0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ ipch Debug Release *.lcl +.vs # Executables *.exe diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index 9eb01e8..e1a3174 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -17,8 +17,13 @@ + + + + + {C7E01836-333F-490D-A3B5-3554A8935040} diff --git a/mod_mrf.vcxproj.filters b/mod_mrf.vcxproj.filters index 2d7f900..d16cfa7 100644 --- a/mod_mrf.vcxproj.filters +++ b/mod_mrf.vcxproj.filters @@ -9,8 +9,8 @@ {93995380-89BD-4b04-88EB-625FBE52EBFB} h;hh;hpp;hxx;hm;inl;inc;xsd - - {a73e6c46-3010-4cc7-b8fa-4d7c875ede0f} + + {76786145-6ccc-4796-bd24-c8edce17e2a3} @@ -19,11 +19,26 @@ - - DOC + + Support Files + + + Support Files + + + Support Files + + + Support Files - DOC + Support Files + + + Support Files + + + Support Files From 8f6718ddc2d87698e7fb05dc5f4cbe45301ca843 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 22 Aug 2018 14:04:20 -0700 Subject: [PATCH 07/85] Minor Makefile changes --- src/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Makefile b/src/Makefile index 04e863b..1dcaa6d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -4,8 +4,9 @@ FILES = $(C_SRC) DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE # Create the ../Makefile.lcl, which should define -# MOD_PATH, the folder where the apache module should reside +# DEST, the folder where the apache module should reside # INCLUDES, the gcc include commands for httpd, apr headers +# LIBTOOL, DEBUG ... include Makefile.lcl TARGET = .libs/mod_mrf.so @@ -13,12 +14,11 @@ TARGET = .libs/mod_mrf.so # Can't use apxs to build c++ modules # The options used here might depend on how apache was built $(TARGET) : $(FILES) - libtool --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(INCLUDES) -pthread -c -o mod_mrf.lo $(C_SRC) && touch mod_mrf.slo - libtool --silent --mode=link g++ -o mod_mrf.la -rpath $(MOD_PATH) -module -avoid-version mod_mrf.lo + $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(INCLUDES) -pthread -c -o mod_mrf.lo $(C_SRC) && touch mod_mrf.slo + $(LIBTOOL) --silent --mode=link g++ -o mod_mrf.la -rpath $(MOD_PATH) -module -avoid-version mod_mrf.lo install : $(TARGET) - sudo cp $(TARGET) $(MOD_PATH) + $(SUDO) cp $(TARGET) $(DEST) clean : rm -rf .libs *.{o,lo,slo,la} - From 59daa9975edac74a53f7258bef5946d982331f6d Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 22 Aug 2018 14:08:09 -0700 Subject: [PATCH 08/85] define debug mode LOG statement --- src/mod_mrf.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/mod_mrf.h b/src/mod_mrf.h index 9defe83..b3ea6b5 100644 --- a/src/mod_mrf.h +++ b/src/mod_mrf.h @@ -20,6 +20,12 @@ #define APR_WANT_MEMFUNC #include +#if defined(DEBUG) +#define LOG(r, frmt, ...) {\ + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, frmt, ##__VA_ARGS__);\ +} +#endif + #define CMD_FUNC (cmd_func) // The maximum size of a tile, to avoid MRF corruption errors From f8c5d9943161c834d29e98da8d60f4f19ad92b6e Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 22 Aug 2018 14:10:28 -0700 Subject: [PATCH 09/85] Use DEST, not MOD_PATH --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 1dcaa6d..e53f045 100644 --- a/src/Makefile +++ b/src/Makefile @@ -15,7 +15,7 @@ TARGET = .libs/mod_mrf.so # The options used here might depend on how apache was built $(TARGET) : $(FILES) $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(INCLUDES) -pthread -c -o mod_mrf.lo $(C_SRC) && touch mod_mrf.slo - $(LIBTOOL) --silent --mode=link g++ -o mod_mrf.la -rpath $(MOD_PATH) -module -avoid-version mod_mrf.lo + $(LIBTOOL) --silent --mode=link g++ -o mod_mrf.la -rpath $(DEST) -module -avoid-version mod_mrf.lo install : $(TARGET) $(SUDO) cp $(TARGET) $(DEST) From e053452ca6b6f85cab2a65076fd8b1931d60828e Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Fri, 2 Nov 2018 13:37:53 -0700 Subject: [PATCH 10/85] Implement retries for source failures Retry fetching data from remote source when the initial request fails. The number to set is in addition to the first fetch. By default it will try 5 times. --- README.md | 3 +++ src/mod_mrf.cpp | 53 +++++++++++++++++++++++++++++++------------------ src/mod_mrf.h | 3 +++ 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index a8cd9fd..56a22c5 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,9 @@ This module takes two apache configuration directives: **Redirect path** - Optional, if the data file is on an object store + **RetryCount N** + - Optional, if the Redirect is also set, how many times to retry retrieving data from the redirect path. Defaults to 4, which means it will try 5 times. Accepts values between 0 and 99. + For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch For better performance when using object stores, the mod_proxy should be patched to reuse connections on subrequests. A patch file is included, see mod_proxy_httpd.patch diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index cd36fc7..e4ab485 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -16,6 +16,7 @@ static void *create_dir_config(apr_pool_t *p, char *dummy) { mrf_conf *c = (mrf_conf *)apr_pcalloc(p, sizeof(mrf_conf)); + c->tries = 5; return c; } @@ -293,8 +294,14 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) } line = apr_table_get(kvp, "Redirect"); - if (line) + if (line) { c->redirect = apr_pstrdup(cmd->pool, line); + line = apr_table_get(kvp, "RetryCount"); + + c->tries = 1 + atoi(line); + if ((c->tries < 1) || (c->tries > 100)) + return "Invalid RetryCount value, should be 0 to 99, defaults to 4"; + } // If we're provided a file name or a size, pre-read the empty tile in the if (efname && (c->datafname == NULL || apr_strnatcmp(c->datafname, efname) || c->esize)) @@ -590,23 +597,31 @@ static int handler(request_rec *r) rctx.maxsize = static_cast(index.size); rctx.size = 0; - request_rec *sr = ap_sub_req_lookup_uri(cfg->redirect, r, r->output_filters); - // Data file is on a remote site a range request redirect with a range header static const char *rfmt = "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT; char *Range = apr_psprintf(r->pool, rfmt, index.offset, index.offset + index.size); - apr_table_setn(sr->headers_in, "Range", Range); - ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, sr, sr->connection); - int status = ap_run_sub_req(sr); - ap_remove_output_filter(rf); - ap_destroy_sub_req(sr); + // S3 may return less than requested, so we retry the request a couple of times + int tries = cfg->tries; + apr_time_t now = apr_time_now(); + do { + request_rec *sr = ap_sub_req_lookup_uri(cfg->redirect, r, r->output_filters); + apr_table_setn(sr->headers_in, "Range", Range); + ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, sr, sr->connection); + int status = ap_run_sub_req(sr); + ap_remove_output_filter(rf); + ap_destroy_sub_req(sr); + + if ((status != APR_SUCCESS || sr->status != HTTP_PARTIAL_CONTENT + || rctx.size != static_cast(index.size)) + && (0 == tries--)) + { // Abort here + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Can't fetch data from %s, took us %" APR_TIME_T_FMT, + cfg->redirect, apr_time_now() - now); + return HTTP_SERVICE_UNAVAILABLE; + } + } while (rctx.size != static_cast(index.size)); - if (status != APR_SUCCESS || sr->status != HTTP_PARTIAL_CONTENT - || rctx.size != static_cast(index.size)) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Can't fetch data from %s", cfg->redirect); - return HTTP_SERVICE_UNAVAILABLE; - } apr_table_clear(r->headers_out); } else @@ -640,19 +655,19 @@ static const command_rec mrf_cmds[] = ), AP_INIT_TAKE1( - "MRF_ConfigurationFile", - CMD_FUNC mrf_file_set, // Callback + "MRF_RegExp", + (cmd_func)set_regexp, 0, // Self-pass argument ACCESS_CONF, // availability - "The configuration file for this module" + "Regular expression that the URL has to match. At least one is required." ), AP_INIT_TAKE1( - "MRF_RegExp", - (cmd_func)set_regexp, + "MRF_ConfigurationFile", + CMD_FUNC mrf_file_set, // Callback 0, // Self-pass argument ACCESS_CONF, // availability - "Regular expression that the URL has to match. At least one is required." + "The configuration file for this module" ), { NULL } diff --git a/src/mod_mrf.h b/src/mod_mrf.h index b3ea6b5..2021396 100644 --- a/src/mod_mrf.h +++ b/src/mod_mrf.h @@ -109,6 +109,9 @@ typedef struct { // Turns the module functionality off int enabled; + // Used on remote data, how many times to try + int tries; + // ETag initializer apr_uint64_t seed; // Buffer for the emtpy tile etag From 0376eeb23d9c810225c4bb38ce70f000c41fcef3 Mon Sep 17 00:00:00 2001 From: lplesea Date: Wed, 19 Dec 2018 09:15:33 +0000 Subject: [PATCH 11/85] Allow debug flag to be set in Makefile.lcl --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index e53f045..0507bfe 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,7 +1,7 @@ C_SRC = mod_mrf.cpp FILES = $(C_SRC) -DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE +DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE $(DEBUG) # Create the ../Makefile.lcl, which should define # DEST, the folder where the apache module should reside From 4551853d7aeb65519b9195d81128524a1652e0ce Mon Sep 17 00:00:00 2001 From: lplesea Date: Wed, 19 Dec 2018 09:45:59 +0000 Subject: [PATCH 12/85] Fixes for parsing input configuration files, error reporting --- src/mod_mrf.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index e4ab485..86456e1 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -152,7 +152,7 @@ static void mrf_init(apr_pool_t *p, mrf_conf *c) { // If present, at least one of them has to match the URL static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) { - char *err_message = NULL; + // char *err_message = NULL; if (c->arr_rxp == 0) c->arr_rxp = apr_array_make(cmd->pool, 2, sizeof(ap_regex_t *)); ap_regex_t **m = (ap_regex_t **)apr_array_push(c->arr_rxp); @@ -293,12 +293,13 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) efname = last; } + line = apr_table_get(kvp, "Redirect"); if (line) { c->redirect = apr_pstrdup(cmd->pool, line); line = apr_table_get(kvp, "RetryCount"); - c->tries = 1 + atoi(line); + c->tries = 1 + (line?atoi(line):0); if ((c->tries < 1) || (c->tries > 100)) return "Invalid RetryCount value, should be 0 to 99, defaults to 4"; } @@ -315,23 +316,23 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) apr_finfo_t finfo; stat = apr_stat(&finfo, efname, APR_FINFO_CSIZE, cmd->temp_pool); if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't stat %s %pm", efname, stat); + return apr_psprintf(cmd->pool, "Can't stat %s %pm", efname, &stat); c->esize = (apr_uint64_t)finfo.csize; } stat = apr_file_open(&efile, efname, APR_FOPEN_READ | APR_FOPEN_BINARY, 0, cmd->temp_pool); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't open empty file %s, loaded from %s: %pm", - efname, arg, stat); + efname, arg, &stat); c->empty = (apr_uint32_t *)apr_palloc(cmd->pool, static_cast(c->esize)); stat = apr_file_seek(efile, APR_SET, &offset); if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't seek empty tile %s: %pm", efname, stat); + return apr_psprintf(cmd->pool, "Can't seek empty tile %s: %pm", efname, &stat); apr_size_t size = (apr_size_t)c->esize; stat = apr_file_read(efile, c->empty, &size); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't read from %s, loaded from %s: %pm", - efname, arg, stat); + efname, arg, &stat); apr_file_close(efile); } From 4340bead30ef932d0431a0e0acf5e4d4699b1ec0 Mon Sep 17 00:00:00 2001 From: lplesea Date: Wed, 19 Dec 2018 10:14:03 +0000 Subject: [PATCH 13/85] More configuration error checking, better error message --- src/mod_mrf.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 86456e1..f3666f2 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -342,6 +342,9 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) c->seed = line ? base32decode(line, &flag) : 0; // Set the missing tile etag, with the extra bit set uint64tobase32(c->seed, c->eETag, 1); + // check configuration sanity + if (!c->idxfname) + return apr_psprintf(cmd->pool, "MRF configuration %s an IndexFile directive", arg); c->enabled = 1; return NULL; } @@ -550,7 +553,7 @@ static int handler(request_rec *r) apr_file_t *idxf, *dataf; SERR_IF(open_index_file(r, &idxf, cfg->idxfname), - apr_psprintf(r->pool, "Can't open %s", cfg->idxfname)); + apr_psprintf(r->pool, "Can't open index %s", cfg->idxfname)); TIdx index; apr_size_t read_size = sizeof(TIdx); From 90c8d9d6836a5094e8b9b701b68b3c12373d497a Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 20 Dec 2018 05:54:51 -0800 Subject: [PATCH 14/85] Add win x64 project --- mod_mrf.sln | 13 +++++++-- mod_mrf.vcxproj | 73 +++++++++++++++++++++++++++++++++++++++++++++++++ src/mod_mrf.cpp | 2 +- 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/mod_mrf.sln b/mod_mrf.sln index 0ad3fcd..5f7196f 100644 --- a/mod_mrf.sln +++ b/mod_mrf.sln @@ -1,22 +1,31 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.40629.0 +# Visual Studio 15 +VisualStudioVersion = 15.0.28307.136 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mod_mrf", "mod_mrf.vcxproj", "{C7E01836-333F-490D-A3B5-3554A8935040}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 Release|Win32 = Release|Win32 + Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|Win32.ActiveCfg = Debug|Win32 {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|Win32.Build.0 = Debug|Win32 + {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|x64.ActiveCfg = Debug|x64 + {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|x64.Build.0 = Debug|x64 {C7E01836-333F-490D-A3B5-3554A8935040}.Release|Win32.ActiveCfg = Release|Win32 {C7E01836-333F-490D-A3B5-3554A8935040}.Release|Win32.Build.0 = Release|Win32 + {C7E01836-333F-490D-A3B5-3554A8935040}.Release|x64.ActiveCfg = Release|x64 + {C7E01836-333F-490D-A3B5-3554A8935040}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {A5643073-62AA-4472-AEC6-7057F30D5683} + EndGlobalSection EndGlobal diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index e1a3174..ce23138 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -5,10 +5,18 @@ Debug Win32 + + Debug + x64 + Release Win32 + + Release + x64 + @@ -38,6 +46,12 @@ v141 NotSet + + DynamicLibrary + true + v141 + NotSet + Application false @@ -45,23 +59,43 @@ true NotSet + + Application + false + v141 + true + NotSet + + + + + + + true .so + + true + .so + false + + false + NotUsing @@ -84,6 +118,28 @@ copy /y $(OutDir)$(TargetName)$(TargetExt) \Apache24\modules + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_WINDOWS;%(PreprocessorDefinitions) + true + \Apache24\include + Default + + + Windows + true + \Apache24\lib + libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) + /EXPORT:mrf_module,@1 + $(OutDir)$(TargetName)$(TargetExt) + + + copy /y $(OutDir)$(TargetName)$(TargetExt) \Apache24\modules + + Level3 @@ -101,6 +157,23 @@ true + + + Level3 + Use + MaxSpeed + true + true + WIN32;NDEBUG;_WINDOWS;%(PreprocessorDefinitions) + true + + + Windows + true + true + true + + diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index f3666f2..74a671b 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -395,7 +395,7 @@ static int send_image(request_rec *r, apr_uint32_t *buffer, apr_size_t size) // ap_set_content_length(r, size); - ap_rwrite(buffer, size, r); + ap_rwrite(buffer, static_cast(size), r); return OK; } From 55746c3960401ef3e7518de36bc5c98bb79bad72 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 18 Feb 2019 16:43:24 -0800 Subject: [PATCH 15/85] minor cleanup --- src/mod_mrf.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 74a671b..83e84c6 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -39,7 +39,6 @@ static apr_array_header_t* tokenize(apr_pool_t *p, const char *s, char sep = '/' static apr_table_t *read_pKVP_from_file(apr_pool_t *pool, const char *fname, char **err_message) { - // Should parse it here and initialize the configuration structure ap_configfile_t *cfg_file; apr_status_t s = ap_pcfg_openfile(&cfg_file, pool, fname); @@ -56,6 +55,9 @@ static apr_table_t *read_pKVP_from_file(apr_pool_t *pool, const char *fname, cha continue; const char *value = buffer; char *key = ap_getword_white(pool, &value); + if (strlen(key) == 0) + continue; + // add allows multiple values with the same key apr_table_add(table, key, value); } @@ -179,7 +181,10 @@ static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size DataFile string - Mandatory, the data file of the MRF. + Mandatory, the data file of the MRF. Or provide a Redirect directive + + Redirect + Instead of reading from the data file, make range requests to this URI IndexFile string Optional, The index file name. @@ -198,9 +203,6 @@ static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) Optional, how many levels to ignore, at the top of the MRF pyramid For example a GCS pyramid will have to skip the one tile level, so this should be 1 - Redirect - Instead of reading from the data file, make range requests to this URI - ETagSeed base32_string Optional, 64 bits in base32 digits. Defaults to 0 The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles @@ -273,7 +275,7 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) if (line) c->skip_levels = atoi(line); - // If an emtpy tile is not provided, it falls through + // If an emtpy tile is not provided, it falls through, which usually results in a 404 error // If provided, it has an optional size and offset followed by file name which defaults to datafile // read the empty tile const char *efname = c->datafname; // Default file name is data file @@ -297,8 +299,8 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) line = apr_table_get(kvp, "Redirect"); if (line) { c->redirect = apr_pstrdup(cmd->pool, line); - line = apr_table_get(kvp, "RetryCount"); + line = apr_table_get(kvp, "RetryCount"); c->tries = 1 + (line?atoi(line):0); if ((c->tries < 1) || (c->tries > 100)) return "Invalid RetryCount value, should be 0 to 99, defaults to 4"; @@ -593,7 +595,7 @@ static int handler(request_rec *r) if (cfg->redirect) { // TODO: S3 authorized requests ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); - SERR_IF(!receive_filter, "Redirect needs mod_receive to be available"); + SERR_IF(!receive_filter, "Using redirect requires mod_receive"); // Get a buffer for the received image receive_ctx rctx; @@ -625,8 +627,6 @@ static int handler(request_rec *r) return HTTP_SERVICE_UNAVAILABLE; } } while (rctx.size != static_cast(index.size)); - - apr_table_clear(r->headers_out); } else { // Read from a local file From 2e6486d78f479dfbc72f5eb37f58adbe6fac676b Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 18 Feb 2019 18:03:41 -0800 Subject: [PATCH 16/85] Changes for split MRF files Untested --- src/mod_mrf.cpp | 130 +++++++++++++++++++++++++++++++++--------------- src/mod_mrf.h | 16 +++--- 2 files changed, 99 insertions(+), 47 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 83e84c6..bf6e20d 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -1,7 +1,7 @@ /* * An OnEarth module that serves tiles from an MRF * Lucian Plesea -* (C) 2016-2017 +* (C) 2016-2018 */ #include "mod_mrf.h" @@ -159,7 +159,30 @@ static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) c->arr_rxp = apr_array_make(cmd->pool, 2, sizeof(ap_regex_t *)); ap_regex_t **m = (ap_regex_t **)apr_array_push(c->arr_rxp); *m = ap_pregcomp(cmd->pool, pattern, 0); - return (NULL != *m) ? NULL : "Bad regular expression"; + return (nullptr != *m) ? nullptr : "Bad regular expression"; +} + +// Parse a comma separated list of sources, add the entries to the array arr, either fnames or redirects +static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_header_t *arr, int fnames = 1) { + apr_array_header_t *inputs = tokenize(cmd->temp_pool, src, ','); + for (int i = 0; i < inputs->nelts; i++) { + source_t *entry = &APR_ARRAY_PUSH(arr, source_t); + memset(entry, 0, sizeof(entry)); + char *input = APR_ARRAY_IDX(inputs, i, char *); + char *fname = ap_getword_white_nc(arr->pool, &input); + if (!fname) + return "Missing source name"; + if (fnames) + entry->datafname = fname; + else + entry->redirect = fname; + // See if there are more arguments, should be offset and size + if (*input != 0) + entry->range.offset = strtoull(input, &input, 0); + if (*input != 0) + entry->range.size = strtoull(input, &input, 0); + } + return nullptr; } /* @@ -244,21 +267,23 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) // Initialize the run-time structures mrf_init(cmd->pool, c); - // The DataFile is optional, if provided the index file is the same thing with the extension removed - line = apr_table_get(kvp, "DataFile"); + if (!c->source) + c->source = apr_array_make(cmd->pool, 1, sizeof(source_t)); - // Data and index in the same location by default - if (line) { // If the data file has a three letter extension, change it to idx for the index - c->datafname = apr_pstrdup(cmd->pool, line); - c->idxfname = apr_pstrdup(cmd->pool, line); - char *last; - char *token = apr_strtok(c->idxfname, ".", &last); // strtok destroys the idxfile - while (*last != 0 && token != NULL) - token = apr_strtok(NULL, ".", &last); - memcpy(c->idxfname, c->datafname, strlen(c->datafname)); // Get a new copy - if (token != NULL && strlen(token) == 3) - memcpy(token, "idx", 3); - } + // The DataFile is alternative with Redirect + if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"))) && + (NULL != (line = parse_sources(cmd, line, c->source)))) + return line; + + // The DataFile is alternative with Redirect + if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "Redirect"))) && + (NULL != (line = parse_sources(cmd, line, c->source, 0)))) + return line; + + line = apr_table_get(kvp, "RetryCount"); + c->tries = 1 + (line ? atoi(line) : 0); + if ((c->tries < 1) || (c->tries > 100)) + return "Invalid RetryCount value, should be 0 to 99, defaults to 4"; // Index file can also be provided line = apr_table_get(kvp, "IndexFile"); @@ -278,7 +303,13 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) // If an emtpy tile is not provided, it falls through, which usually results in a 404 error // If provided, it has an optional size and offset followed by file name which defaults to datafile // read the empty tile - const char *efname = c->datafname; // Default file name is data file + // Default file name is the name of the first data file, if provided + const char *datafname = NULL; + for (int i = 0; i < c->source->nelts; i++) + if (NULL != (datafname = APR_ARRAY_IDX(c->source, i, source_t).datafname)) + break; + + const char *efname = datafname; line = apr_table_get(kvp, "EmptyTile"); if (line) { char *last; @@ -295,19 +326,8 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) efname = last; } - - line = apr_table_get(kvp, "Redirect"); - if (line) { - c->redirect = apr_pstrdup(cmd->pool, line); - - line = apr_table_get(kvp, "RetryCount"); - c->tries = 1 + (line?atoi(line):0); - if ((c->tries < 1) || (c->tries > 100)) - return "Invalid RetryCount value, should be 0 to 99, defaults to 4"; - } - // If we're provided a file name or a size, pre-read the empty tile in the - if (efname && (c->datafname == NULL || apr_strnatcmp(c->datafname, efname) || c->esize)) + if (efname && (datafname == NULL || apr_strnatcmp(datafname, efname) || c->esize)) { apr_file_t *efile; apr_off_t offset = c->eoffset; @@ -344,9 +364,21 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) c->seed = line ? base32decode(line, &flag) : 0; // Set the missing tile etag, with the extra bit set uint64tobase32(c->seed, c->eETag, 1); - // check configuration sanity - if (!c->idxfname) - return apr_psprintf(cmd->pool, "MRF configuration %s an IndexFile directive", arg); + + // Set the index file name based on the first data file, if there is only one + if (!c->idxfname) { + if (!datafname) + return "Missing IndexFile or DataFile directive"; + c->idxfname = apr_pstrdup(cmd->pool, datafname); + char *last; + char *token = apr_strtok(c->idxfname, ".", &last); // strtok destroys the idxfile + while (*last != 0 && token != NULL) + token = apr_strtok(NULL, ".", &last); + memcpy(c->idxfname, datafname, strlen(datafname)); // Get a new copy + if (token != NULL && strlen(token) == 3) + memcpy(token, "idx", 3); + } + c->enabled = 1; return NULL; } @@ -511,6 +543,20 @@ static bool our_request(request_rec *r, mrf_conf *cfg) { } +// Return the first source which contains the index, adjusts the index offset if necessary +static const source_t *get_source(const apr_array_header_t *sources, TIdx *index) { + for (int i = 0; i < sources->nelts; i++) { + source_t *source = &APR_ARRAY_IDX(sources, i, source_t); + if ((source->range.offset == 0 && source->range.size == 0) || + (index->offset > source->range.offset && + (source->range.offset - index->offset + index->size) <= source->range.size)) { + index->offset -= source->range.offset; + return source; + } + } + return NULL; +} + static int handler(request_rec *r) { // Only get and no arguments @@ -576,7 +622,7 @@ static int handler(request_rec *r) return HTTP_INTERNAL_SERVER_ERROR; } - // Check for conditional ETag here, no need to open the data file + // Check for conditional ETag here, no need to get the data char ETag[16]; // Try to distribute the bits a bit to generate an ETag uint64tobase32(cfg->seed ^ (index.size << 40) ^ index.offset, ETag); @@ -586,13 +632,15 @@ static int handler(request_rec *r) } // Now for the data part - if (!cfg->datafname && !cfg->redirect) + const source_t *src = get_source(cfg->source, &index); + + if (!src || (!src->datafname && !src->redirect)) SERR_IF(true, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); apr_uint32_t *buffer = static_cast( apr_palloc(r->pool, static_cast(index.size))); - if (cfg->redirect) { + if (src->redirect) { // TODO: S3 authorized requests ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); SERR_IF(!receive_filter, "Using redirect requires mod_receive"); @@ -611,7 +659,7 @@ static int handler(request_rec *r) int tries = cfg->tries; apr_time_t now = apr_time_now(); do { - request_rec *sr = ap_sub_req_lookup_uri(cfg->redirect, r, r->output_filters); + request_rec *sr = ap_sub_req_lookup_uri(src->redirect, r, r->output_filters); apr_table_setn(sr->headers_in, "Range", Range); ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, sr, sr->connection); int status = ap_run_sub_req(sr); @@ -623,24 +671,24 @@ static int handler(request_rec *r) && (0 == tries--)) { // Abort here ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Can't fetch data from %s, took us %" APR_TIME_T_FMT, - cfg->redirect, apr_time_now() - now); + src->redirect, apr_time_now() - now); return HTTP_SERVICE_UNAVAILABLE; } } while (rctx.size != static_cast(index.size)); } else { // Read from a local file - SERR_IF(open_data_file(r, &dataf, cfg->datafname), - apr_psprintf(r->pool, "Can't open %s", cfg->datafname)); + SERR_IF(open_data_file(r, &dataf, src->datafname), + apr_psprintf(r->pool, "Can't open %s", src->datafname)); // We got the tile index, and is not empty SERR_IF(!buffer, "Memory allocation error in mod_mrf"); SERR_IF(apr_file_seek(dataf, APR_SET, (apr_off_t *)&index.offset), - apr_psprintf(r->pool, "Seek error in %s", cfg->datafname)); + apr_psprintf(r->pool, "Seek error in %s", src->datafname)); read_size = static_cast(index.size); SERR_IF(apr_file_read(dataf, buffer, &read_size) || read_size != index.size, - apr_psprintf(r->pool, "Can't read from %s", cfg->datafname)); + apr_psprintf(r->pool, "Can't read from %s", src->datafname)); } // Looks fine, set the outgoing etag and then the image diff --git a/src/mod_mrf.h b/src/mod_mrf.h index 2021396..e21786b 100644 --- a/src/mod_mrf.h +++ b/src/mod_mrf.h @@ -1,7 +1,7 @@ /* * mod_mrf header file * Lucian Plesea -* (C) 2016 +* (C) 2016-2018 */ #if !defined(MOD_MRF_H) @@ -81,11 +81,18 @@ typedef struct { apr_uint64_t size; } TIdx; +// A data source, either local file or a redirect +// the range, if not 0,0, holds the valid offset ranges in this source +typedef struct { + char *redirect; + char *datafname; + TIdx range; +} source_t; + typedef struct { // array of guard regexp, one of them has to match apr_array_header_t *arr_rxp; - // The mrf data file name - char *datafname; + apr_array_header_t *source; // The mrf index file name char *idxfname; // Forced mime-type, default is autodetected @@ -116,9 +123,6 @@ typedef struct { apr_uint64_t seed; // Buffer for the emtpy tile etag char eETag[16]; - // The internal redirect path or null - char *redirect; - } mrf_conf; extern module AP_MODULE_DECLARE_DATA mrf_module; From 3740562ceedc2558237e626746dfc122be858671 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 19 Feb 2019 14:05:15 -0800 Subject: [PATCH 17/85] Fix source size and range check --- src/mod_mrf.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index bf6e20d..8140dc4 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -547,9 +547,11 @@ static bool our_request(request_rec *r, mrf_conf *cfg) { static const source_t *get_source(const apr_array_header_t *sources, TIdx *index) { for (int i = 0; i < sources->nelts; i++) { source_t *source = &APR_ARRAY_IDX(sources, i, source_t); - if ((source->range.offset == 0 && source->range.size == 0) || - (index->offset > source->range.offset && - (source->range.offset - index->offset + index->size) <= source->range.size)) { + if ((source->range.offset == 0 && source->range.size == 0) + || (index->offset >= source->range.offset + && (source->range.size == 0 + || index->offset - source->range.offset + index->size <= source->range.size))) + { index->offset -= source->range.offset; return source; } From 929459c20f19faafd7d0af758a1ff5afb993e6fa Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 19 Feb 2019 14:09:00 -0800 Subject: [PATCH 18/85] Fix memset call Is this even necessary? --- src/mod_mrf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 8140dc4..39bce06 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -167,7 +167,7 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_head apr_array_header_t *inputs = tokenize(cmd->temp_pool, src, ','); for (int i = 0; i < inputs->nelts; i++) { source_t *entry = &APR_ARRAY_PUSH(arr, source_t); - memset(entry, 0, sizeof(entry)); + memset(entry, 0, sizeof(source_t)); char *input = APR_ARRAY_IDX(inputs, i, char *); char *fname = ap_getword_white_nc(arr->pool, &input); if (!fname) From 491928de90cc994ace0b5c69ac1a9b98af8b0836 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 19 Feb 2019 15:06:09 -0800 Subject: [PATCH 19/85] Add Indirect directive When set, the module only responds to internal subrequests --- README.md | 3 +++ src/mod_mrf.cpp | 19 +++++++++++++++++-- src/mod_mrf.h | 2 ++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 56a22c5..497c805 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ This module takes two apache configuration directives: **RetryCount N** - Optional, if the Redirect is also set, how many times to retry retrieving data from the redirect path. Defaults to 4, which means it will try 5 times. Accepts values between 0 and 99. + **Indirect ** + - Optional, the module will only honor subrequests if this is set + For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch For better performance when using object stores, the mod_proxy should be patched to reuse connections on subrequests. A patch file is included, see mod_proxy_httpd.patch diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 39bce06..4c4e61c 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -185,6 +185,12 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_head return nullptr; } +static int get_bool(const char *s) { + while (*s != 0 && (*s == ' ' || *s == '\t')) + s++; + return (!ap_cstr_casecmp(s, "On") || !ap_cstr_casecmp(s, "1")); +} + /* Read the configuration file, which is a key-value text file, with one key per line comment lines that start with # @@ -230,6 +236,9 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_head Optional, 64 bits in base32 digits. Defaults to 0 The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles have ETags that depend on this one and bit 64 is zero + + Indirect + Optional, the module will only honor subrequests if set */ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) @@ -300,6 +309,10 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) if (line) c->skip_levels = atoi(line); + // Check the indirect flag + if (nullptr != (line = apr_table_get(kvp, "Indirect"))) + c->indirect = get_bool(line); + // If an emtpy tile is not provided, it falls through, which usually results in a 404 error // If provided, it has an optional size and offset followed by file name which defaults to datafile // read the empty tile @@ -562,12 +575,14 @@ static const source_t *get_source(const apr_array_header_t *sources, TIdx *index static int handler(request_rec *r) { // Only get and no arguments - if (r->args) return DECLINED; // Don't accept arguments + if (r->args || r->method_number != M_GET) return DECLINED; // Don't accept arguments mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) ? (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) : (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); - if (!cfg->enabled || !our_request(r, cfg)) return DECLINED; + + if (!cfg->enabled || (cfg->indirect && !r->main) || !our_request(r, cfg)) + return DECLINED; apr_array_header_t *tokens = tokenize(r->pool, r->uri, '/'); if (tokens->nelts < 3) return DECLINED; // At least Level Row Column diff --git a/src/mod_mrf.h b/src/mod_mrf.h index e21786b..36db82d 100644 --- a/src/mod_mrf.h +++ b/src/mod_mrf.h @@ -115,6 +115,8 @@ typedef struct { // Turns the module functionality off int enabled; + // If set, only secondary requests are allowed + int indirect; // Used on remote data, how many times to try int tries; From 5ec0d1b719df3a061b8642f700c633efbd932916 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 20 Feb 2019 11:21:12 -0800 Subject: [PATCH 20/85] Elevated Indirect flag to httpd configuration Makes more sense to have the indirect flag in the httpd configuration. Also moved the index read in a separate function --- README.md | 7 +++-- src/mod_mrf.cpp | 73 ++++++++++++++++++++++++++++--------------------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 497c805..2ea9e82 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ This module takes two apache configuration directives: Defaults to on if the MRF_ConfigurationFile is provided + **MRF_Indirect On|Off** + + If set, this module will only respond to internal subrequests + **MRF_ConfigurationFile Filename** Points to a text file that contains lines, where the first word on a line is a directive, followed by parameters @@ -54,9 +58,6 @@ This module takes two apache configuration directives: **RetryCount N** - Optional, if the Redirect is also set, how many times to retry retrieving data from the redirect path. Defaults to 4, which means it will try 5 times. Accepts values between 0 and 99. - **Indirect ** - - Optional, the module will only honor subrequests if this is set - For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch For better performance when using object stores, the mod_proxy should be patched to reuse connections on subrequests. A patch file is included, see mod_proxy_httpd.patch diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 4c4e61c..604ed48 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -12,6 +12,10 @@ using namespace std; +#if APR_SUCCESS != 0 +#error "APR_SUCCESS is not null" +#endif + static void *create_dir_config(apr_pool_t *p, char *dummy) { mrf_conf *c = @@ -185,12 +189,6 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_head return nullptr; } -static int get_bool(const char *s) { - while (*s != 0 && (*s == ' ' || *s == '\t')) - s++; - return (!ap_cstr_casecmp(s, "On") || !ap_cstr_casecmp(s, "1")); -} - /* Read the configuration file, which is a key-value text file, with one key per line comment lines that start with # @@ -237,8 +235,6 @@ static int get_bool(const char *s) { The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles have ETags that depend on this one and bit 64 is zero - Indirect - Optional, the module will only honor subrequests if set */ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) @@ -309,10 +305,6 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) if (line) c->skip_levels = atoi(line); - // Check the indirect flag - if (nullptr != (line = apr_table_get(kvp, "Indirect"))) - c->indirect = get_bool(line); - // If an emtpy tile is not provided, it falls through, which usually results in a 404 error // If provided, it has an optional size and offset followed by file name which defaults to datafile // read the empty tile @@ -487,7 +479,7 @@ static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, cons // Use the connection pool for the note and file, to ensure it gets closed with the connection apr_pool_t *pool = r->connection->pool; - if (fn != NULL) { // We have an file note but it is not the right file + if (fn != NULL) { // We have a file note but it is not the right file apr_table_unset(conn_notes, note_name); // Unhook the existing note apr_file_close(fn->pfh); // Close the existing file } @@ -572,6 +564,23 @@ static const source_t *get_source(const apr_array_header_t *sources, TIdx *index return NULL; } +static const char *read_index(request_rec *r, TIdx *idx, apr_off_t offset) { + mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module); + apr_file_t *idxf; + if (open_index_file(r, &idxf, cfg->idxfname)) + return apr_psprintf(r->pool, "Can't open index %s", cfg->idxfname); + + apr_size_t read_size = sizeof(TIdx); + if (apr_file_seek(idxf, APR_SET, &offset) + || apr_file_read(idxf, &idx, &read_size) + || read_size != sizeof(TIdx)) + return apr_psprintf(r->pool, "Invalid read in %s", cfg->idxfname); + + idx->offset = ntoh64(idx->offset); + idx->size = ntoh64(idx->size); + return nullptr; +} + static int handler(request_rec *r) { // Only get and no arguments @@ -616,21 +625,12 @@ static int handler(request_rec *r) apr_off_t tidx_offset = level->offset + sizeof(TIdx) * (tile.x + level->width * (tile.z * level->height + tile.y)); - apr_file_t *idxf, *dataf; - SERR_IF(open_index_file(r, &idxf, cfg->idxfname), - apr_psprintf(r->pool, "Can't open index %s", cfg->idxfname)); TIdx index; - apr_size_t read_size = sizeof(TIdx); - - SERR_IF(apr_file_seek(idxf, APR_SET, &tidx_offset) - || apr_file_read(idxf, &index, &read_size) - || read_size != sizeof(TIdx), - apr_psprintf(r->pool, "Tile index doesn't exist in %s", cfg->idxfname)); + const char *message; + SERR_IF((message = read_index(r, &index, tidx_offset)), + message); // MRF index record is in network order - index.size = ntoh64(index.size); - index.offset = ntoh64(index.offset); - if (index.size < 4) // Need at least four bytes for signature check return send_empty_tile(r); @@ -656,6 +656,8 @@ static int handler(request_rec *r) apr_uint32_t *buffer = static_cast( apr_palloc(r->pool, static_cast(index.size))); + SERR_IF(!buffer, + "Memory allocation error in mod_mrf"); if (src->redirect) { // TODO: S3 authorized requests @@ -669,8 +671,8 @@ static int handler(request_rec *r) rctx.size = 0; // Data file is on a remote site a range request redirect with a range header - static const char *rfmt = "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT; - char *Range = apr_psprintf(r->pool, rfmt, index.offset, index.offset + index.size); + char *Range = apr_psprintf(r->pool, "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT, + index.offset, index.offset + index.size); // S3 may return less than requested, so we retry the request a couple of times int tries = cfg->tries; @@ -687,7 +689,8 @@ static int handler(request_rec *r) || rctx.size != static_cast(index.size)) && (0 == tries--)) { // Abort here - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Can't fetch data from %s, took us %" APR_TIME_T_FMT, + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Can't fetch data from %s, took us %" APR_TIME_T_FMT, src->redirect, apr_time_now() - now); return HTTP_SERVICE_UNAVAILABLE; } @@ -695,15 +698,15 @@ static int handler(request_rec *r) } else { // Read from a local file + apr_file_t *dataf; SERR_IF(open_data_file(r, &dataf, src->datafname), apr_psprintf(r->pool, "Can't open %s", src->datafname)); // We got the tile index, and is not empty - SERR_IF(!buffer, - "Memory allocation error in mod_mrf"); SERR_IF(apr_file_seek(dataf, APR_SET, (apr_off_t *)&index.offset), apr_psprintf(r->pool, "Seek error in %s", src->datafname)); - read_size = static_cast(index.size); + + apr_size_t read_size = static_cast(index.size); SERR_IF(apr_file_read(dataf, buffer, &read_size) || read_size != index.size, apr_psprintf(r->pool, "Can't read from %s", src->datafname)); } @@ -723,6 +726,14 @@ static const command_rec mrf_cmds[] = "mod_mrf enable, defaults to on if configuration is provided" ), + AP_INIT_FLAG( + "MRF_Indirect", + CMD_FUNC ap_set_flag_slot, + (void *)APR_OFFSETOF(mrf_conf, indirect), + ACCESS_CONF, + "If set, this configuration only responds to subrequests" + ), + AP_INIT_TAKE1( "MRF_RegExp", (cmd_func)set_regexp, From dbc120a1e1e647b7461cbef7cfa0f3c0f1292f5e Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 20 Feb 2019 11:24:51 -0800 Subject: [PATCH 21/85] Corrected documentation MRF_RegExp appears in the apache configuration, not in the control file --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2ea9e82..daea554 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ This module takes two apache configuration directives: Defaults to on if the MRF_ConfigurationFile is provided + **MRF_RegExp** + + Required, only requests matching this pattern are handled. It can appear multiple times + **MRF_Indirect On|Off** If set, this module will only respond to internal subrequests @@ -24,9 +28,6 @@ This module takes two apache configuration directives: **DataFile string** - Mandatory, the path to the data file of the MRF. - **RegExp** - - Optional, a regular expression that must match the request URL for it to be considered a tile request. By default all URLs are considered tile requests. This directive can appear multiple times. If there are multiple RegExp lines, at least one has to match the request URL. - **PageSize X Y 1 C** - Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size From 1f7f4996e79db71829c91a430f42289708ee9cd6 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 20 Feb 2019 11:37:52 -0800 Subject: [PATCH 22/85] Update README.md --- README.md | 59 +++++++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index daea554..de48449 100644 --- a/README.md +++ b/README.md @@ -3,61 +3,56 @@ An apache module that serves tiles directly from a local MRF, 2D or 3D. This module takes two apache configuration directives: - **MRF On|Off** +**MRF On|Off** Defaults to on if the MRF_ConfigurationFile is provided - **MRF_RegExp** +**MRF_RegExp** Required, only requests matching this pattern are handled. It can appear multiple times - **MRF_Indirect On|Off** +**MRF_Indirect On|Off** If set, this module will only respond to internal subrequests - **MRF_ConfigurationFile Filename** +**MRF_ConfigurationFile Filename** Points to a text file that contains lines, where the first word on a line is a directive, followed by parameters - Empty lines, lines that start with # are considered comments - Unknown directives are ignored - Known directives for this module are: - **Size X Y Z C** - - Mandatory, at least x and y, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (these are usually not meaningful) +***Size X Y Z C*** + - Mandatory, at least x and y, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (these are usually not meaningful) - **DataFile string** - - Mandatory, the path to the data file of the MRF. +***DataFile string start_offset size*** + - The path to the MRF data file. Can appear multiple times, with start offset and size values. Start offset and size default to zero. Zero size means that the data size is unlimited. At least one DataFile or Redirect directive is required. - **PageSize X Y 1 C** - - Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size +***Redirect path start_offset size*** + - Optional, the path where the tile range request should be made. Can appear multiple times, with start offset and size values. Start offset and size default to zero. Zero size means that the data size is unlimited. - **IndexFile string** - - Optional, The index file name. - If not provided it uses the data file name if its extension is not three letters. - Otherwise it uses the datafile name with the extension changed to .idx +***RetryCount N*** + - Optional, if the Redirect is also set, how many times to retry retrieving data from the redirect path. Defaults to 4, which means it will try 5 times. Accepts values between 0 and 99. + +***PageSize X Y 1 C*** + - Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size + +***IndexFile string*** + - Optional, the index file name. If not provided it uses the data file name if its extension is not three letters. + Otherwise it uses the first data file name with the extension changed to .idx - **MimeType string** - - Optional. Defaults to autodetect. +***MimeType string*** + - Optional, defaults to autodetect. - **EmptyTile Size Offset FileName** - - Optional. By default it ignores the request if a tile is missing. - First number is assumed to be the size, second is offset. - If filename is not provided, it uses the data file name. +***EmptyTile Size Offset FileName*** + - Optional, provides the tile content to be sent when the requested tile is missing. By default the request is ignored, which results in a 404 error if a fallback mechanism does not exist. if present, the first number is assumed to be the size, second is offset. If filename is not given, the first data file name is used. - **SkippedLevels N** - - Optional, how many levels to ignore, at the top of the MRF pyramid. - For example a GCS pyramid will have to skip the one tile level, so this should be 1 +***SkippedLevels N*** + - Optional, how many levels to ignore, at the top of the MRF pyramid. For example a GCS pyramid will have to skip the one tile level, so this should be 1 - **ETagSeed base32_string** - - Optional, 64 bits in base32 digits. Defaults to 0. - The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles - have 64 bit ETags that depend on this value. +***ETagSeed base32_string*** + - Optional, 64 bits in base32 digits, defaults to 0. The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles have 64 bit ETags that depend on this value. - **Redirect path** - - Optional, if the data file is on an object store - - **RetryCount N** - - Optional, if the Redirect is also set, how many times to retry retrieving data from the redirect path. Defaults to 4, which means it will try 5 times. Accepts values between 0 and 99. For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch From 306c046e6993c4d50641093a8b3946c04b6c5a72 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 20 Feb 2019 11:39:24 -0800 Subject: [PATCH 23/85] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de48449..3b4cb74 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,11 @@ This module takes two apache configuration directives: **MRF_ConfigurationFile Filename** - Points to a text file that contains lines, where the first word on a line is a directive, followed by parameters + Points to an AHTSE Control text file, where the first word on a line is a directive, followed by parameters - Empty lines, lines that start with # are considered comments - Unknown directives are ignored - - Known directives for this module are: + +AHTSE Control Directives for this module are: ***Size X Y Z C*** - Mandatory, at least x and y, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (these are usually not meaningful) From 784b47616344b8f6d1d7dacf956261e2ab6bbf58 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 20 Feb 2019 12:50:33 -0800 Subject: [PATCH 24/85] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3b4cb74..5cc9155 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ This module takes two apache configuration directives: - Empty lines, lines that start with # are considered comments - Unknown directives are ignored -AHTSE Control Directives for this module are: +AHTSE Control Directives for thiccs module are: ***Size X Y Z C*** - Mandatory, at least x and y, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (these are usually not meaningful) @@ -52,7 +52,7 @@ AHTSE Control Directives for this module are: - Optional, how many levels to ignore, at the top of the MRF pyramid. For example a GCS pyramid will have to skip the one tile level, so this should be 1 ***ETagSeed base32_string*** - - Optional, 64 bits in base32 digits, defaults to 0. The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles have 64 bit ETags that depend on this value. + - Optional, 64 bits as 13 base32 digits, defaults to 0. The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles have 64 bit ETags that depend on this value. For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch From 94c68a0d98c9e124f68074ff0cc99e72fe7509ef Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 20 Feb 2019 12:51:18 -0800 Subject: [PATCH 25/85] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cc9155..e0ae05f 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ AHTSE Control Directives for thiccs module are: - Optional, how many levels to ignore, at the top of the MRF pyramid. For example a GCS pyramid will have to skip the one tile level, so this should be 1 ***ETagSeed base32_string*** - - Optional, 64 bits as 13 base32 digits, defaults to 0. The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles have 64 bit ETags that depend on this value. + - Optional, 64 bits as 13 base32 digits [0-9a-v], defaults to 0. The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles have 64 bit ETags that depend on this value. For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch From 522b4d1f0d9dc5d67956ab15402397838e3e24ed Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 20 Feb 2019 12:53:39 -0800 Subject: [PATCH 26/85] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e0ae05f..571d470 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ AHTSE Control Directives for thiccs module are: - Optional, how many levels to ignore, at the top of the MRF pyramid. For example a GCS pyramid will have to skip the one tile level, so this should be 1 ***ETagSeed base32_string*** - - Optional, 64 bits as 13 base32 digits [0-9a-v], defaults to 0. The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles have 64 bit ETags that depend on this value. + - Optional, 64 bits as 13 base32 digits [0-9a-v], defaults to 0. The empty tile ETag will be this value but 65th bit is set, also the only value that has this bit set. All the other tiles have 64 bit ETags that depend on this value. For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch From 9d1bf42fbe07bd83f44f632fef3fa2f34211820a Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Fri, 1 Mar 2019 13:48:12 -0800 Subject: [PATCH 27/85] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 571d470..6e3c334 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# mod_mrf +# mod_mrf [AHTSE](https://github.com/lucianpls/AHTSE) An apache module that serves tiles directly from a local MRF, 2D or 3D. This module takes two apache configuration directives: From 92a63ddbaf756638d49963e51ee972852f84dd73 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 11 Apr 2019 14:27:27 -0700 Subject: [PATCH 28/85] consistent use of module configuration --- src/mod_mrf.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 604ed48..3bd38e9 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -395,9 +395,7 @@ static int etag_matches(request_rec *r, const char *ETag) { static int send_image(request_rec *r, apr_uint32_t *buffer, apr_size_t size) { - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - ? (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - : (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); + mrf_conf *cfg = get_conf(r); if (cfg->mime_type) ap_set_content_type(r, cfg->mime_type); else @@ -440,9 +438,7 @@ static int send_image(request_rec *r, apr_uint32_t *buffer, apr_size_t size) // Returns the empty tile if defined static int send_empty_tile(request_rec *r) { - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - ? (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - : (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); + mrf_conf *cfg = get_conf(r); if (etag_matches(r, cfg->eETag)) { apr_table_setn(r->headers_out, "ETag", cfg->eETag); return HTTP_NOT_MODIFIED; @@ -547,6 +543,12 @@ static bool our_request(request_rec *r, mrf_conf *cfg) { return false; } +// Use the request config if it exists, otherwise use directory config +static inline mrf_conf * get_conf(request_rec *r) { + mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module); + if (cfg) return cfg; + return (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); +} // Return the first source which contains the index, adjusts the index offset if necessary static const source_t *get_source(const apr_array_header_t *sources, TIdx *index) { @@ -565,7 +567,8 @@ static const source_t *get_source(const apr_array_header_t *sources, TIdx *index } static const char *read_index(request_rec *r, TIdx *idx, apr_off_t offset) { - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module); + mrf_conf *cfg = get_conf(r); + apr_file_t *idxf; if (open_index_file(r, &idxf, cfg->idxfname)) return apr_psprintf(r->pool, "Can't open index %s", cfg->idxfname); @@ -586,10 +589,7 @@ static int handler(request_rec *r) // Only get and no arguments if (r->args || r->method_number != M_GET) return DECLINED; // Don't accept arguments - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - ? (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module) - : (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); - + mrf_conf *cfg = get_conf(r); if (!cfg->enabled || (cfg->indirect && !r->main) || !our_request(r, cfg)) return DECLINED; From af5e17efe14ee3036a1817a2eb7251e66403cf51 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 11 Apr 2019 14:49:17 -0700 Subject: [PATCH 29/85] Fix read_index function --- src/mod_mrf.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 3bd38e9..e0c1928 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -16,7 +16,7 @@ using namespace std; #error "APR_SUCCESS is not null" #endif -static void *create_dir_config(apr_pool_t *p, char *dummy) +static inline void *create_dir_config(apr_pool_t *p, char *dummy) { mrf_conf *c = (mrf_conf *)apr_pcalloc(p, sizeof(mrf_conf)); @@ -24,6 +24,13 @@ static void *create_dir_config(apr_pool_t *p, char *dummy) return c; } +// Use the request config if it exists, otherwise use directory config +static inline mrf_conf * get_conf(request_rec *r) { + mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module); + if (cfg) return cfg; + return (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); +} + // // Tokenize a string into an array // @@ -543,13 +550,6 @@ static bool our_request(request_rec *r, mrf_conf *cfg) { return false; } -// Use the request config if it exists, otherwise use directory config -static inline mrf_conf * get_conf(request_rec *r) { - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module); - if (cfg) return cfg; - return (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); -} - // Return the first source which contains the index, adjusts the index offset if necessary static const source_t *get_source(const apr_array_header_t *sources, TIdx *index) { for (int i = 0; i < sources->nelts; i++) { @@ -575,7 +575,7 @@ static const char *read_index(request_rec *r, TIdx *idx, apr_off_t offset) { apr_size_t read_size = sizeof(TIdx); if (apr_file_seek(idxf, APR_SET, &offset) - || apr_file_read(idxf, &idx, &read_size) + || apr_file_read(idxf, idx, &read_size) || read_size != sizeof(TIdx)) return apr_psprintf(r->pool, "Invalid read in %s", cfg->idxfname); From 3864ed16a6a1723ecf8ecab71c52309bc5bf2d4a Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 16 Apr 2019 13:47:01 -0700 Subject: [PATCH 30/85] Deprecate Redirect directive Use DataFile that starts with : instead --- README.md | 37 ++++++++++++++++++++++++----------- src/mod_mrf.cpp | 51 +++++++++++++++++++++++++++++++------------------ src/mod_mrf.h | 6 +++--- 3 files changed, 61 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 6e3c334..fd6bf70 100644 --- a/README.md +++ b/README.md @@ -21,26 +21,39 @@ This module takes two apache configuration directives: - Empty lines, lines that start with # are considered comments - Unknown directives are ignored -AHTSE Control Directives for thiccs module are: +AHTSE Control Directives for this module are: + +***DataFile path start_offset size*** + - The path to the MRF data file to serve tiles from. Start and size are optional, by default + a single DataFile is used. At least one DataFile directive is required. If the path start + with colon (:) followed by slash /, the path is interpreted as an internal redirect to a path + within the same server, starting from DocRoot. Otherwise it is assumed to be a local file + name. May appear multiple times, with different start offset and size values. If the values are + present, read operations within start_offset and start_offset + size are made to the data file, after the read + offset is adjusted downward by start_offset. + If the read offset falls outside the range, the other DataFile entries are searched, in the order in which they + appear in the configuration file, but first all local files will be checked and then the redirects. This allows an MRF data file to be split into multiple parts. Single tiles + cannot be split, but overlapping ranges are allowed, the first match will be used. Only one read + operation will be issued, to the first DataFile entry that matches the range. If the read fails, + the server will report an error. + Start offset and size default to zero. Zero size means that any read above the offset will be done in + this data file. ***Size X Y Z C*** - Mandatory, at least x and y, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (these are usually not meaningful) -***DataFile string start_offset size*** - - The path to the MRF data file. Can appear multiple times, with start offset and size values. Start offset and size default to zero. Zero size means that the data size is unlimited. At least one DataFile or Redirect directive is required. - -***Redirect path start_offset size*** - - Optional, the path where the tile range request should be made. Can appear multiple times, with start offset and size values. Start offset and size default to zero. Zero size means that the data size is unlimited. - -***RetryCount N*** - - Optional, if the Redirect is also set, how many times to retry retrieving data from the redirect path. Defaults to 4, which means it will try 5 times. Accepts values between 0 and 99. - ***PageSize X Y 1 C*** - Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size +***RetryCount N*** + - Optional, [0 - 99). If the DataFiles are redirects, how many times to attempt retrieving data from + the source path. Defaults to 4, which means it will try 5 times. + ***IndexFile string*** - - Optional, the index file name. If not provided it uses the data file name if its extension is not three letters. + - Optional, the index file name. Can only be provided once. + If not provided it uses the data file name if its extension is not three letters. Otherwise it uses the first data file name with the extension changed to .idx + It can be a redirect path in the host namespace, if it starts with a colon : ***MimeType string*** - Optional, defaults to autodetect. @@ -54,6 +67,8 @@ AHTSE Control Directives for thiccs module are: ***ETagSeed base32_string*** - Optional, 64 bits as 13 base32 digits [0-9a-v], defaults to 0. The empty tile ETag will be this value but 65th bit is set, also the only value that has this bit set. All the other tiles have 64 bit ETags that depend on this value. +***Redirect path start_offset size*** + *Deprecated*, use the DataFile directive and start path with : For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index e0c1928..bb93a20 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -180,13 +180,24 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_head source_t *entry = &APR_ARRAY_PUSH(arr, source_t); memset(entry, 0, sizeof(source_t)); char *input = APR_ARRAY_IDX(inputs, i, char *); + char *fname = ap_getword_white_nc(arr->pool, &input); if (!fname) return "Missing source name"; - if (fnames) - entry->datafname = fname; - else + + if (fnames) { + if (strlen(fname) > 3 && fname[0] == ':' && fname[1] == '/') { + // this is a new style redirect entry + entry->redirect = fname + 1; + } + else { + entry->fname = fname; + } + } + else { entry->redirect = fname; + } + // See if there are more arguments, should be offset and size if (*input != 0) entry->range.offset = strtoull(input, &input, 0); @@ -299,8 +310,9 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) // Index file can also be provided line = apr_table_get(kvp, "IndexFile"); - if (line) - c->idxfname = apr_pstrdup(cmd->pool, line); + if (line) { + c->idx.fname = apr_pstrdup(cmd->pool, line); + } // Mime type is autodetected if not provided line = apr_table_get(kvp, "MimeType"); @@ -318,7 +330,7 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) // Default file name is the name of the first data file, if provided const char *datafname = NULL; for (int i = 0; i < c->source->nelts; i++) - if (NULL != (datafname = APR_ARRAY_IDX(c->source, i, source_t).datafname)) + if (NULL != (datafname = APR_ARRAY_IDX(c->source, i, source_t).fname)) break; const char *efname = datafname; @@ -378,15 +390,15 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) uint64tobase32(c->seed, c->eETag, 1); // Set the index file name based on the first data file, if there is only one - if (!c->idxfname) { + if (!c->idx.fname) { if (!datafname) return "Missing IndexFile or DataFile directive"; - c->idxfname = apr_pstrdup(cmd->pool, datafname); + c->idx.fname = apr_pstrdup(cmd->pool, datafname); char *last; - char *token = apr_strtok(c->idxfname, ".", &last); // strtok destroys the idxfile + char *token = apr_strtok(c->idx.fname, ".", &last); // strtok destroys the idxfile while (*last != 0 && token != NULL) token = apr_strtok(NULL, ".", &last); - memcpy(c->idxfname, datafname, strlen(datafname)); // Get a new copy + memcpy(c->idx.fname, datafname, strlen(datafname)); // Get a new copy if (token != NULL && strlen(token) == 3) memcpy(token, "idx", 3); } @@ -570,14 +582,14 @@ static const char *read_index(request_rec *r, TIdx *idx, apr_off_t offset) { mrf_conf *cfg = get_conf(r); apr_file_t *idxf; - if (open_index_file(r, &idxf, cfg->idxfname)) - return apr_psprintf(r->pool, "Can't open index %s", cfg->idxfname); + if (open_index_file(r, &idxf, cfg->idx.fname)) + return apr_psprintf(r->pool, "Can't open index %s", cfg->idx.fname); apr_size_t read_size = sizeof(TIdx); if (apr_file_seek(idxf, APR_SET, &offset) || apr_file_read(idxf, idx, &read_size) || read_size != sizeof(TIdx)) - return apr_psprintf(r->pool, "Invalid read in %s", cfg->idxfname); + return apr_psprintf(r->pool, "Invalid read in %s", cfg->idx.fname); idx->offset = ntoh64(idx->offset); idx->size = ntoh64(idx->size); @@ -635,7 +647,8 @@ static int handler(request_rec *r) return send_empty_tile(r); if (MAX_TILE_SIZE < index.size) { // Tile is too large, log and send error code - ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, "Tile too large in %s", cfg->idxfname); + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, "Tile too large in %s", + cfg->idx.fname); return HTTP_INTERNAL_SERVER_ERROR; } @@ -651,7 +664,7 @@ static int handler(request_rec *r) // Now for the data part const source_t *src = get_source(cfg->source, &index); - if (!src || (!src->datafname && !src->redirect)) + if (!src || (!src->fname && !src->redirect)) SERR_IF(true, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); apr_uint32_t *buffer = static_cast( @@ -699,16 +712,16 @@ static int handler(request_rec *r) else { // Read from a local file apr_file_t *dataf; - SERR_IF(open_data_file(r, &dataf, src->datafname), - apr_psprintf(r->pool, "Can't open %s", src->datafname)); + SERR_IF(open_data_file(r, &dataf, src->fname), + apr_psprintf(r->pool, "Can't open %s", src->fname)); // We got the tile index, and is not empty SERR_IF(apr_file_seek(dataf, APR_SET, (apr_off_t *)&index.offset), - apr_psprintf(r->pool, "Seek error in %s", src->datafname)); + apr_psprintf(r->pool, "Seek error in %s", src->fname)); apr_size_t read_size = static_cast(index.size); SERR_IF(apr_file_read(dataf, buffer, &read_size) || read_size != index.size, - apr_psprintf(r->pool, "Can't read from %s", src->datafname)); + apr_psprintf(r->pool, "Can't read from %s", src->fname)); } // Looks fine, set the outgoing etag and then the image diff --git a/src/mod_mrf.h b/src/mod_mrf.h index 36db82d..eabb845 100644 --- a/src/mod_mrf.h +++ b/src/mod_mrf.h @@ -85,7 +85,7 @@ typedef struct { // the range, if not 0,0, holds the valid offset ranges in this source typedef struct { char *redirect; - char *datafname; + char *fname; TIdx range; } source_t; @@ -93,8 +93,8 @@ typedef struct { // array of guard regexp, one of them has to match apr_array_header_t *arr_rxp; apr_array_header_t *source; - // The mrf index file name - char *idxfname; + source_t idx; + // Forced mime-type, default is autodetected char *mime_type; // Full raster size in pixels From 83c92e308922092c7fa1e4efa98d795fcf061b35 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 16 Apr 2019 15:41:07 -0700 Subject: [PATCH 31/85] a few more changes This module needs a complete rewrite, based on libahtse --- src/mod_mrf.cpp | 61 +++++++++++++++++++++++++++---------------------- src/mod_mrf.h | 6 ++--- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index bb93a20..6743f76 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -186,15 +186,14 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_head return "Missing source name"; if (fnames) { - if (strlen(fname) > 3 && fname[0] == ':' && fname[1] == '/') { - // this is a new style redirect entry - entry->redirect = fname + 1; - } - else { + if (strlen(fname) > 3 && fname[0] == ':' && fname[1] == '/') + entry->redirect = fname + 1; // new style redirect + + else entry->fname = fname; - } } else { + // Old style redirect entry->redirect = fname; } @@ -308,10 +307,13 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) if ((c->tries < 1) || (c->tries > 100)) return "Invalid RetryCount value, should be 0 to 99, defaults to 4"; - // Index file can also be provided + // Index file can also be provided, there could be a default line = apr_table_get(kvp, "IndexFile"); if (line) { - c->idx.fname = apr_pstrdup(cmd->pool, line); + if (strlen(line) > 2 && line[0] == ':' && line[1] == '/') + c->idx.redirect = apr_pstrdup(cmd->pool, line + 1); + else + c->idx.fname = apr_pstrdup(cmd->pool, line); } // Mime type is autodetected if not provided @@ -478,15 +480,14 @@ static const apr_int32_t open_flags = APR_FOPEN_READ | APR_FOPEN_BINARY | APR_FO /* * Open or retrieve an connection cached file. */ -static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, const char *name, - apr_int32_t flags = open_flags, const char *note_name = "MRF_INDEX_FILE") - +static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, + const source_t &src, const char *note_name, apr_int32_t flags = open_flags) { apr_table_t *conn_notes = r->connection->notes; // Try to pick it up from the connection notes file_note *fn = (file_note *) apr_table_get(conn_notes, note_name); - if ((fn != NULL) && !apr_strnatcmp(name, fn->name)) { // Match, set file and return + if ((fn != NULL) && !apr_strnatcmp(src.fname, fn->name)) { // Match, set file and return *ppfh = fn->pfh; return APR_SUCCESS; } @@ -502,30 +503,31 @@ static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, cons fn = (file_note *)apr_palloc(pool, sizeof(file_note)); } - apr_status_t stat = apr_file_open(ppfh, name, flags, 0, pool); + apr_status_t stat = apr_file_open(ppfh, src.fname, flags, 0, pool); if (stat != APR_SUCCESS) return stat; // Fill the note and hook it up, then return fn->pfh = *ppfh; - fn->name = apr_pstrdup(pool, name); // The old string will persist until cleaned by the pool + fn->name = apr_pstrdup(pool, src.fname); // The old string will persist until cleaned by the pool apr_table_setn(conn_notes, note_name, (const char *) fn); return APR_SUCCESS; } -#define open_index_file open_connection_file - // Open data file optimized for random access if possible -static apr_status_t open_data_file(request_rec *r, apr_file_t **ppfh, const char *name) +static apr_status_t open_data_file(request_rec *r, apr_file_t **ppfh, + const source_t &src) { static const char data_note_name[] = "MRF_DATA_FILE"; #if defined(APR_FOPEN_RANDOM) // apr has portable support for random access to files - return open_connection_file(r, ppfh, name, open_flags | APR_FOPEN_RANDOM, data_note_name); + return open_connection_file(r, ppfh, src, open_flags | APR_FOPEN_RANDOM, + data_note_name); #else - apr_status_t stat = open_connection_file(r, ppfh, name, open_flags, data_note_name); + apr_status_t stat = + open_connection_file(r, ppfh, src, data_note_name, open_flags); #if !defined(POSIX_FADV_RANDOM) return stat; @@ -580,16 +582,18 @@ static const source_t *get_source(const apr_array_header_t *sources, TIdx *index static const char *read_index(request_rec *r, TIdx *idx, apr_off_t offset) { mrf_conf *cfg = get_conf(r); - apr_file_t *idxf; - if (open_index_file(r, &idxf, cfg->idx.fname)) - return apr_psprintf(r->pool, "Can't open index %s", cfg->idx.fname); + + if (open_connection_file(r, &idxf, cfg->idx, "MRF_INDEX_FILE")) + return apr_psprintf(r->pool, + "Can't open index %s", cfg->idx.fname); apr_size_t read_size = sizeof(TIdx); if (apr_file_seek(idxf, APR_SET, &offset) || apr_file_read(idxf, idx, &read_size) || read_size != sizeof(TIdx)) - return apr_psprintf(r->pool, "Invalid read in %s", cfg->idx.fname); + return apr_psprintf(r->pool, + "%s : Read error", cfg->idx.fname); idx->offset = ntoh64(idx->offset); idx->size = ntoh64(idx->size); @@ -599,14 +603,16 @@ static const char *read_index(request_rec *r, TIdx *idx, apr_off_t offset) { static int handler(request_rec *r) { // Only get and no arguments - if (r->args || r->method_number != M_GET) return DECLINED; // Don't accept arguments + if (r->args || r->method_number != M_GET) + return DECLINED; mrf_conf *cfg = get_conf(r); if (!cfg->enabled || (cfg->indirect && !r->main) || !our_request(r, cfg)) return DECLINED; apr_array_header_t *tokens = tokenize(r->pool, r->uri, '/'); - if (tokens->nelts < 3) return DECLINED; // At least Level Row Column + if (tokens->nelts < 3) + return DECLINED; // At least Level Row Column // Use a xyzc structure, with c being the level // Input order is M/Level/Row/Column, with M being optional @@ -693,7 +699,8 @@ static int handler(request_rec *r) do { request_rec *sr = ap_sub_req_lookup_uri(src->redirect, r, r->output_filters); apr_table_setn(sr->headers_in, "Range", Range); - ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, sr, sr->connection); + ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, + sr, sr->connection); int status = ap_run_sub_req(sr); ap_remove_output_filter(rf); ap_destroy_sub_req(sr); @@ -712,7 +719,7 @@ static int handler(request_rec *r) else { // Read from a local file apr_file_t *dataf; - SERR_IF(open_data_file(r, &dataf, src->fname), + SERR_IF(open_data_file(r, &dataf, *src), apr_psprintf(r->pool, "Can't open %s", src->fname)); // We got the tile index, and is not empty diff --git a/src/mod_mrf.h b/src/mod_mrf.h index eabb845..457ec34 100644 --- a/src/mod_mrf.h +++ b/src/mod_mrf.h @@ -68,12 +68,10 @@ struct sz { apr_int64_t x, y, z, c, l; }; +// width and height are in tiles struct rset { apr_off_t offset; - // in tiles - int width; - // in tiles - int height; + int width, height; }; typedef struct { From 1c9fd1ddff104c4c58757eba833b3866d9d5ae05 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 16 Apr 2019 17:48:22 -0700 Subject: [PATCH 32/85] Compiles against libahtse --- mod_mrf.vcxproj | 3 +- src/mod_mrf.cpp | 170 +++++++++++++----------------------------------- src/mod_mrf.h | 101 +++------------------------- 3 files changed, 55 insertions(+), 219 deletions(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index ce23138..18fd235 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -85,6 +85,7 @@ true .so + \Apache24\include;$(VC_SourcePath); true @@ -110,7 +111,7 @@ Windows true \Apache24\lib - libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) + libahtse.lib;libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) /EXPORT:mrf_module,@1 $(OutDir)$(TargetName)$(TargetExt) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 6743f76..d7093e7 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -1,7 +1,7 @@ /* * An OnEarth module that serves tiles from an MRF * Lucian Plesea -* (C) 2016-2018 +* (C) 2016-2019 */ #include "mod_mrf.h" @@ -9,11 +9,17 @@ #include #include +#include +#include + +#define CMD_FUNC (cmd_func) using namespace std; -#if APR_SUCCESS != 0 -#error "APR_SUCCESS is not null" +NS_AHTSE_USE + +#if defined(APLOG_USE_MODULE) +APLOG_USE_MODULE(mrf); #endif static inline void *create_dir_config(apr_pool_t *p, char *dummy) @@ -31,21 +37,6 @@ static inline mrf_conf * get_conf(request_rec *r) { return (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); } -// -// Tokenize a string into an array -// -static apr_array_header_t* tokenize(apr_pool_t *p, const char *s, char sep = '/') -{ - apr_array_header_t* arr = apr_array_make(p, 10, sizeof(char *)); - while (sep == *s) s++; - char *val; - while (*s && (val = ap_getword(p, &s, sep))) { - char **newelt = (char **)apr_array_push(arr); - *newelt = val; - } - return arr; -} - // Returns a table read from a file, or NULL and an error message static apr_table_t *read_pKVP_from_file(apr_pool_t *pool, const char *fname, char **err_message) @@ -82,7 +73,9 @@ static apr_table_t *read_pKVP_from_file(apr_pool_t *pool, const char *fname, cha } // Returns NULL if it worked as expected, returns a four integer value from "x y", "x y z" or "x y z c" -static char *get_xyzc_size(apr_pool_t *p, struct sz *size, const char *value, const char*err_prefix) { +static char *get_xyzc_size(apr_pool_t *p, sz *size, + const char *value, const char*err_prefix) +{ char *s; if (!value) return apr_psprintf(p, "%s directive missing", err_prefix); @@ -100,48 +93,13 @@ static char *get_xyzc_size(apr_pool_t *p, struct sz *size, const char *value, co return NULL; } -// Converts a 64bit value into 13 trigesimal chars -static void uint64tobase32(apr_uint64_t value, char *buffer, int flag = 0) { - static char b32digits[] = "0123456789abcdefghijklmnopqrstuv"; - // From the bottom up - buffer[13] = 0; // End of string marker - for (int i = 0; i < 12; i++, value >>= 5) - buffer[12 - i] = b32digits[value & 0x1f]; - // First char holds the empty tile flag - if (flag) flag = 0x10; // Making sure it has the right value - buffer[0] = b32digits[flag | value]; -} - -// Return the value from a base 32 character -// Returns a negative value if char is not a valid base32 char -static int b32(char ic) { - int c = 0xff & (static_cast(ic)); - if (c < '0') return -1; - if (c - '0' < 10) return c - '0'; - if (c < 'A') return -1; - if (c - 'A' < 22) return c - 'A' + 10; - if (c < 'a') return -1; - if (c - 'a' < 22) return c - 'a' + 10; - return -1; -} - -static apr_uint64_t base32decode(const char *is, int *flag) { - apr_int64_t value = 0; - const unsigned char *s = reinterpret_cast(is); - while (*s == '"') s++; // Skip initial quotes - *flag = (b32(*s) >> 4) & 1; // Pick up the flag from bit 5 - for (int v = b32(*s++) & 0xf; v >= 0; v = b32(*s++)) - value = (value << 5) + v; - return value; -} - static void mrf_init(apr_pool_t *p, mrf_conf *c) { - struct rset level; - level.width = static_cast(1 + (c->size.x - 1) / c->pagesize.x); - level.height = static_cast(1 + (c->size.y - 1) / c->pagesize.y); + rset level; + level.w = static_cast(1 + (c->size.x - 1) / c->pagesize.x); + level.h = static_cast(1 + (c->size.y - 1) / c->pagesize.y); level.offset = 0; // How many levels do we have - c->n_levels = 2 + ilogb(max(level.height, level.width) - 1); + c->n_levels = 2 + ilogb(max(level.h, level.w) - 1); c->rsets = (struct rset *)apr_pcalloc(p, sizeof(rset) * c->n_levels); // Populate rsets from the bottom, the way tile protcols count levels @@ -153,12 +111,12 @@ static void mrf_init(apr_pool_t *p, mrf_conf *c) { // This is safe on all platforms that have large files (64bit signed offset) // It will start failing if the file offset is larger than 63bits // The c->size.z has to be first, to force the 64bit type - level.offset += c->size.z * level.width * level.height * sizeof(TIdx); - level.width = 1 + (level.width - 1) / 2; - level.height = 1 + (level.height - 1) / 2; + level.offset += c->size.z * level.w * level.h * sizeof(range_t); + level.w = 1 + (level.w - 1) / 2; + level.h = 1 + (level.h - 1) / 2; } // MRF has one tile at the top - ap_assert(c->rsets->height == 1 && c->rsets->width == 1); + ap_assert(c->rsets->h == 1 && c->rsets->w == 1); } // Allow for one or more RegExp guard @@ -340,7 +298,7 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) if (line) { char *last; // Try to read a figure first - c->esize = apr_strtoi64(line, &last, 0); + c->empty.size = (int)apr_strtoi64(line, &last, 0); // If that worked, try to get an offset too if (last != line) @@ -353,31 +311,32 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) } // If we're provided a file name or a size, pre-read the empty tile in the - if (efname && (datafname == NULL || apr_strnatcmp(datafname, efname) || c->esize)) + if (efname && + (datafname == NULL || apr_strnatcmp(datafname, efname) || c->empty.size)) { apr_file_t *efile; apr_off_t offset = c->eoffset; apr_status_t stat; // Use the temp pool for the file open, it will close it for us - if (!c->esize) { // Don't know the size, get it from the file + if (!c->empty.size) { // Don't know the size, get it from the file apr_finfo_t finfo; stat = apr_stat(&finfo, efname, APR_FINFO_CSIZE, cmd->temp_pool); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't stat %s %pm", efname, &stat); - c->esize = (apr_uint64_t)finfo.csize; + c->empty.size = (int)finfo.csize; } stat = apr_file_open(&efile, efname, APR_FOPEN_READ | APR_FOPEN_BINARY, 0, cmd->temp_pool); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't open empty file %s, loaded from %s: %pm", efname, arg, &stat); - c->empty = (apr_uint32_t *)apr_palloc(cmd->pool, static_cast(c->esize)); + c->empty.buffer = (char *)apr_palloc(cmd->pool, static_cast(c->empty.size)); stat = apr_file_seek(efile, APR_SET, &offset); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't seek empty tile %s: %pm", efname, &stat); - apr_size_t size = (apr_size_t)c->esize; - stat = apr_file_read(efile, c->empty, &size); + apr_size_t size = c->empty.size; + stat = apr_file_read(efile, (void *)c->empty.buffer, &size); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't read from %s, loaded from %s: %pm", efname, arg, &stat); @@ -389,7 +348,7 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) int flag; c->seed = line ? base32decode(line, &flag) : 0; // Set the missing tile etag, with the extra bit set - uint64tobase32(c->seed, c->eETag, 1); + tobase32(c->seed, c->eETag, 1); // Set the index file name based on the first data file, if there is only one if (!c->idx.fname) { @@ -414,49 +373,6 @@ static int etag_matches(request_rec *r, const char *ETag) { return ETagIn != 0 && strstr(ETagIn, ETag); } -static int send_image(request_rec *r, apr_uint32_t *buffer, apr_size_t size) -{ - mrf_conf *cfg = get_conf(r); - if (cfg->mime_type) - ap_set_content_type(r, cfg->mime_type); - else - switch (hton32(*buffer)) { - case JPEG_SIG: - ap_set_content_type(r, "image/jpeg"); - break; - case PNG_SIG: - ap_set_content_type(r, "image/png"); - break; - default: // LERC goes here too - ap_set_content_type(r, "application/octet-stream"); - } - - // Is it gzipped content? - if (GZIP_SIG == hton32(*buffer)) { - apr_table_setn(r->headers_out, "Content-Encoding", "gzip"); - const char *ae = apr_table_get(r->headers_in, "Accept-Encoding"); - // If accept encoding is missing, assume it doesn't support gzip - if (!ae || !strstr(ae, "gzip")) { - ap_filter_rec_t *inflate_filter = ap_get_output_filter_handle("INFLATE"); - if (inflate_filter) - ap_add_output_filter_handle(inflate_filter, NULL, r, r->connection); - else - ap_log_rerror(APLOG_MARK, APLOG_WARNING, 0, r, - "mod_deflate not loaded, sending gzipped tile to client that does not declare support"); - } - } - - // - // Static headers can be added with the header module - // apr_table_setn(r->headers_out, "Access-Control-Allow-Origin", "*"); - // apt_table_setn(r->headers_out, "Cache-Control", "max-age=86400"); - // - - ap_set_content_length(r, size); - ap_rwrite(buffer, static_cast(size), r); - return OK; -} - // Returns the empty tile if defined static int send_empty_tile(request_rec *r) { mrf_conf *cfg = get_conf(r); @@ -465,8 +381,9 @@ static int send_empty_tile(request_rec *r) { return HTTP_NOT_MODIFIED; } - if (!cfg->empty) return DECLINED; // Passthrough - return send_image(r, cfg->empty, static_cast(cfg->esize)); + if (!cfg->empty.buffer) + return DECLINED; // Passthrough + return sendImage(r, cfg->empty); } // An open file handle and the matching file name, to be used as a note @@ -565,7 +482,7 @@ static bool our_request(request_rec *r, mrf_conf *cfg) { } // Return the first source which contains the index, adjusts the index offset if necessary -static const source_t *get_source(const apr_array_header_t *sources, TIdx *index) { +static const source_t *get_source(const apr_array_header_t *sources, range_t *index) { for (int i = 0; i < sources->nelts; i++) { source_t *source = &APR_ARRAY_IDX(sources, i, source_t); if ((source->range.offset == 0 && source->range.size == 0) @@ -580,7 +497,7 @@ static const source_t *get_source(const apr_array_header_t *sources, TIdx *index return NULL; } -static const char *read_index(request_rec *r, TIdx *idx, apr_off_t offset) { +static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { mrf_conf *cfg = get_conf(r); apr_file_t *idxf; @@ -588,15 +505,15 @@ static const char *read_index(request_rec *r, TIdx *idx, apr_off_t offset) { return apr_psprintf(r->pool, "Can't open index %s", cfg->idx.fname); - apr_size_t read_size = sizeof(TIdx); + apr_size_t read_size = sizeof(range_t); if (apr_file_seek(idxf, APR_SET, &offset) || apr_file_read(idxf, idx, &read_size) - || read_size != sizeof(TIdx)) + || read_size != sizeof(range_t)) return apr_psprintf(r->pool, "%s : Read error", cfg->idx.fname); - idx->offset = ntoh64(idx->offset); - idx->size = ntoh64(idx->size); + idx->offset = be64toh(idx->offset); + idx->size = be64toh(idx->size); return nullptr; } @@ -637,13 +554,13 @@ static int handler(request_rec *r) // Check for bad requests, outside of the defined bounds REQ_ERR_IF(tile.l >= cfg->n_levels); rset *level = cfg->rsets + tile.l; - REQ_ERR_IF(tile.x >= level->width || tile.y >= level->height); + REQ_ERR_IF(tile.x >= level->w || tile.y >= level->h); // Offset of the index entry for this tile apr_off_t tidx_offset = level->offset + - sizeof(TIdx) * (tile.x + level->width * (tile.z * level->height + tile.y)); + sizeof(range_t) * (tile.x + level->w * (tile.z * level->h + tile.y)); - TIdx index; + range_t index; const char *message; SERR_IF((message = read_index(r, &index, tidx_offset)), message); @@ -661,7 +578,7 @@ static int handler(request_rec *r) // Check for conditional ETag here, no need to get the data char ETag[16]; // Try to distribute the bits a bit to generate an ETag - uint64tobase32(cfg->seed ^ (index.size << 40) ^ index.offset, ETag); + tobase32(cfg->seed ^ (index.size << 40) ^ index.offset, ETag); if (etag_matches(r, ETag)) { apr_table_set(r->headers_out, "ETag", ETag); return HTTP_NOT_MODIFIED; @@ -733,7 +650,8 @@ static int handler(request_rec *r) // Looks fine, set the outgoing etag and then the image apr_table_set(r->headers_out, "ETag", ETag); - return send_image(r, buffer, static_cast(index.size)); + storage_manager temp = { (char *)buffer, static_cast(index.size) }; + return sendImage(r, temp); } static const command_rec mrf_cmds[] = diff --git a/src/mod_mrf.h b/src/mod_mrf.h index 457ec34..d50de9b 100644 --- a/src/mod_mrf.h +++ b/src/mod_mrf.h @@ -1,114 +1,31 @@ /* * mod_mrf header file * Lucian Plesea -* (C) 2016-2018 +* (C) 2016-2019 */ #if !defined(MOD_MRF_H) -#include -#include -#include -#include -#include -#include -#include -#include +#include -#include - -#define APR_WANT_STRFUNC -#define APR_WANT_MEMFUNC -#include - -#if defined(DEBUG) -#define LOG(r, frmt, ...) {\ - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, frmt, ##__VA_ARGS__);\ -} -#endif - -#define CMD_FUNC (cmd_func) - -// The maximum size of a tile, to avoid MRF corruption errors -#define MAX_TILE_SIZE 4*1024*1024 - -// signatures in big endian, to autodetect tile type -#define PNG_SIG 0x89504e47 -#define JPEG_SIG 0xffd8ffe0 -#define LERC_SIG 0x436e745a - -// This one is not a type, just an encoding -#define GZIP_SIG 0x1f8b0800 - -// Conversion to and from network order, endianess depenent - -#if (APR_IS_BIGENDIAN == 0) // Little endian -#if defined(WIN32) // Windows -#define ntoh32(v) _byteswap_ulong(v) -#define hton32(v) _byteswap_ulong(v) -#define ntoh64(v) _byteswap_uint64(v) -#define hton64(v) _byteswap_uint64(v) -#else // Assume linux -#define ntoh32(v) __builtin_bswap32(v) -#define hton32(v) __builtin_bswap32(v) -#define ntoh64(v) __builtin_bswap64(v) -#define hton64(v) __builtin_bswap64(v) -#endif -#else // Big endian, do nothing -#define ntoh32(v) (v) -#define ntoh64(v) (v) -#define hton32(v) (v) -#define hton64(v) (v) -#endif - -#if defined(APLOG_USE_MODULE) -APLOG_USE_MODULE(mrf); -#endif - -struct sz { - apr_int64_t x, y, z, c, l; -}; - -// width and height are in tiles -struct rset { - apr_off_t offset; - int width, height; -}; - -typedef struct { - apr_uint64_t offset; - apr_uint64_t size; -} TIdx; - -// A data source, either local file or a redirect -// the range, if not 0,0, holds the valid offset ranges in this source -typedef struct { - char *redirect; - char *fname; - TIdx range; -} source_t; - -typedef struct { +struct mrf_conf { // array of guard regexp, one of them has to match apr_array_header_t *arr_rxp; apr_array_header_t *source; - source_t idx; + AHTSE::source_t idx; // Forced mime-type, default is autodetected char *mime_type; // Full raster size in pixels - struct sz size; + AHTSE::sz size; // Page size in pixels - struct sz pagesize; + AHTSE::sz pagesize; // Levels to skip at the top int skip_levels; int n_levels; - struct rset *rsets; + AHTSE::rset *rsets; - // Empty tile buffer, if provided - apr_uint32_t *empty; - // Size of empty tile, in bytes - apr_int64_t esize; + AHTSE::storage_manager empty; apr_off_t eoffset; // Turns the module functionality off @@ -123,7 +40,7 @@ typedef struct { apr_uint64_t seed; // Buffer for the emtpy tile etag char eETag[16]; -} mrf_conf; +}; extern module AP_MODULE_DECLARE_DATA mrf_module; From 61a82bf213976c3a9fde41a114ef739204fce982 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 16 Apr 2019 18:02:00 -0700 Subject: [PATCH 33/85] Removed header Single source file module --- mod_mrf.vcxproj | 3 --- mod_mrf.vcxproj.filters | 5 ----- src/mod_mrf.cpp | 39 +++++++++++++++++++++++++++++++++- src/mod_mrf.h | 47 ----------------------------------------- 4 files changed, 38 insertions(+), 56 deletions(-) delete mode 100644 src/mod_mrf.h diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index 18fd235..279961d 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -21,9 +21,6 @@ - - - diff --git a/mod_mrf.vcxproj.filters b/mod_mrf.vcxproj.filters index d16cfa7..5bbe540 100644 --- a/mod_mrf.vcxproj.filters +++ b/mod_mrf.vcxproj.filters @@ -13,11 +13,6 @@ {76786145-6ccc-4796-bd24-c8edce17e2a3} - - - Header Files - - Support Files diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index d7093e7..903c99d 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -4,7 +4,7 @@ * (C) 2016-2019 */ -#include "mod_mrf.h" +#include #include "receive_context.h" #include @@ -18,6 +18,43 @@ using namespace std; NS_AHTSE_USE +struct mrf_conf { + // array of guard regexp, one of them has to match + apr_array_header_t *arr_rxp; + apr_array_header_t *source; + source_t idx; + + // Forced mime-type, default is autodetected + char *mime_type; + // Full raster size in pixels + sz size; + // Page size in pixels + sz pagesize; + + // Levels to skip at the top + int skip_levels; + int n_levels; + rset *rsets; + + storage_manager empty; + apr_off_t eoffset; + + // Turns the module functionality off + int enabled; + // If set, only secondary requests are allowed + int indirect; + + // Used on remote data, how many times to try + int tries; + + // ETag initializer + apr_uint64_t seed; + // Buffer for the emtpy tile etag + char eETag[16]; +}; + +extern module AP_MODULE_DECLARE_DATA mrf_module; + #if defined(APLOG_USE_MODULE) APLOG_USE_MODULE(mrf); #endif diff --git a/src/mod_mrf.h b/src/mod_mrf.h deleted file mode 100644 index d50de9b..0000000 --- a/src/mod_mrf.h +++ /dev/null @@ -1,47 +0,0 @@ -/* -* mod_mrf header file -* Lucian Plesea -* (C) 2016-2019 -*/ - -#if !defined(MOD_MRF_H) -#include - -struct mrf_conf { - // array of guard regexp, one of them has to match - apr_array_header_t *arr_rxp; - apr_array_header_t *source; - AHTSE::source_t idx; - - // Forced mime-type, default is autodetected - char *mime_type; - // Full raster size in pixels - AHTSE::sz size; - // Page size in pixels - AHTSE::sz pagesize; - - // Levels to skip at the top - int skip_levels; - int n_levels; - AHTSE::rset *rsets; - - AHTSE::storage_manager empty; - apr_off_t eoffset; - - // Turns the module functionality off - int enabled; - // If set, only secondary requests are allowed - int indirect; - - // Used on remote data, how many times to try - int tries; - - // ETag initializer - apr_uint64_t seed; - // Buffer for the emtpy tile etag - char eETag[16]; -}; - -extern module AP_MODULE_DECLARE_DATA mrf_module; - -#endif From 1b7c7321b2c14cbc0c57955695dff27b6b30d14e Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sat, 20 Apr 2019 15:39:43 -0700 Subject: [PATCH 34/85] Use libahtse more --- src/mod_mrf.cpp | 274 +++++++++++++----------------------------------- 1 file changed, 70 insertions(+), 204 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 903c99d..2df5a0e 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -22,7 +22,7 @@ struct mrf_conf { // array of guard regexp, one of them has to match apr_array_header_t *arr_rxp; apr_array_header_t *source; - source_t idx; + vfile_t idx; // Forced mime-type, default is autodetected char *mime_type; @@ -36,7 +36,7 @@ struct mrf_conf { int n_levels; rset *rsets; - storage_manager empty; + empty_conf_t empty; apr_off_t eoffset; // Turns the module functionality off @@ -67,48 +67,6 @@ static inline void *create_dir_config(apr_pool_t *p, char *dummy) return c; } -// Use the request config if it exists, otherwise use directory config -static inline mrf_conf * get_conf(request_rec *r) { - mrf_conf *cfg = (mrf_conf *)ap_get_module_config(r->request_config, &mrf_module); - if (cfg) return cfg; - return (mrf_conf *)ap_get_module_config(r->per_dir_config, &mrf_module); -} - -// Returns a table read from a file, or NULL and an error message -static apr_table_t *read_pKVP_from_file(apr_pool_t *pool, const char *fname, char **err_message) - -{ - ap_configfile_t *cfg_file; - apr_status_t s = ap_pcfg_openfile(&cfg_file, pool, fname); - - if (APR_SUCCESS != s) { // %pm means print status error string - *err_message = apr_psprintf(pool, "%s - %pm", fname, &s); - return NULL; - } - - char buffer[MAX_STRING_LEN]; - apr_table_t *table = apr_table_make(pool, 8); - // This can return ENOSPC if lines are too long - while (APR_SUCCESS == (s = ap_cfg_getline(buffer, MAX_STRING_LEN, cfg_file))) { - if ((strlen(buffer) == 0) || buffer[0] == '#') - continue; - const char *value = buffer; - char *key = ap_getword_white(pool, &value); - if (strlen(key) == 0) - continue; - // add allows multiple values with the same key - apr_table_add(table, key, value); - } - - ap_cfg_closefile(cfg_file); - if (s == APR_ENOSPC) { - *err_message = apr_psprintf(pool, "%s lines should be smaller than %d", fname, MAX_STRING_LEN); - return NULL; - } - - return table; -} - // Returns NULL if it worked as expected, returns a four integer value from "x y", "x y z" or "x y z c" static char *get_xyzc_size(apr_pool_t *p, sz *size, const char *value, const char*err_prefix) @@ -158,7 +116,8 @@ static void mrf_init(apr_pool_t *p, mrf_conf *c) { // Allow for one or more RegExp guard // If present, at least one of them has to match the URL -static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) +static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, + const char *pattern) { // char *err_message = NULL; if (c->arr_rxp == 0) @@ -168,93 +127,42 @@ static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) return (nullptr != *m) ? nullptr : "Bad regular expression"; } -// Parse a comma separated list of sources, add the entries to the array arr, either fnames or redirects -static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_header_t *arr, int fnames = 1) { +// Parse a comma separated list of sources, add the entries to the array arr +// Source may include offset and size, white space separated +static const char *parse_sources(cmd_parms *cmd, const char *src, + apr_array_header_t *arr, bool redir = false) +{ apr_array_header_t *inputs = tokenize(cmd->temp_pool, src, ','); for (int i = 0; i < inputs->nelts; i++) { - source_t *entry = &APR_ARRAY_PUSH(arr, source_t); - memset(entry, 0, sizeof(source_t)); + vfile_t *entry = &APR_ARRAY_PUSH(arr, vfile_t); + memset(entry, 0, sizeof(vfile_t)); char *input = APR_ARRAY_IDX(inputs, i, char *); char *fname = ap_getword_white_nc(arr->pool, &input); if (!fname) return "Missing source name"; - if (fnames) { - if (strlen(fname) > 3 && fname[0] == ':' && fname[1] == '/') - entry->redirect = fname + 1; // new style redirect - - else - entry->fname = fname; - } - else { - // Old style redirect - entry->redirect = fname; - } + entry->name = fname; // See if there are more arguments, should be offset and size - if (*input != 0) - entry->range.offset = strtoull(input, &input, 0); - if (*input != 0) - entry->range.size = strtoull(input, &input, 0); + if (*input != 0) entry->range.offset = strtoull(input, &input, 0); + if (*input != 0) entry->range.size = strtoull(input, &input, 0); } return nullptr; } -/* - Read the configuration file, which is a key-value text file, with one key per line - comment lines that start with # - empty lines are allowed, as well as continued lines if the first one ends with \ - However, every line is limited to the Apache max string size, defaults to 8192 chars - - Unknown keys, or keys that are misspelled are silently ignored - Keys are not case sensitive, but values are - Keys and values are space separated. The last value per line, if it is a string, may contain spaces - - Supported keys: - - Size X Y - Mandatory, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (usually not meaningful) - - PageSize X Y <1> - Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size - - DataFile string - Mandatory, the data file of the MRF. Or provide a Redirect directive - - Redirect - Instead of reading from the data file, make range requests to this URI - - IndexFile string - Optional, The index file name. - If not provided it uses the data file name if its extension is not three letters. - Otherwise it uses the datafile name with the extension changed to .idx - - MimeType string - Optional. Defaults to autodetect - - EmptyTile - Optional. By default it ignores the request if a tile is missing - First number is assumed to be the size, second is offset - If filename is not provided, it uses the data file name - - SkippedLevels - Optional, how many levels to ignore, at the top of the MRF pyramid - For example a GCS pyramid will have to skip the one tile level, so this should be 1 - - ETagSeed base32_string - Optional, 64 bits in base32 digits. Defaults to 0 - The empty tile ETag will be this value but bit 64 (65th bit) is set. All the other tiles - have ETags that depend on this one and bit 64 is zero - - */ +static const char *parse_redirects(cmd_parms *cmd, const char *src, + apr_array_header_t *arr) +{ + return parse_sources(cmd, src, arr, true); +} static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) { ap_assert(sizeof(apr_off_t) == 8); mrf_conf *c = (mrf_conf *)dconf; - char *err_message; - apr_table_t *kvp = read_pKVP_from_file(cmd->temp_pool, arg, &err_message); + const char *err_message; + apr_table_t *kvp = readAHTSEConfig(cmd->temp_pool, arg, &err_message); if (NULL == kvp) return err_message; // Got the parsed kvp table, parse the configuration items @@ -285,16 +193,16 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) mrf_init(cmd->pool, c); if (!c->source) - c->source = apr_array_make(cmd->pool, 1, sizeof(source_t)); + c->source = apr_array_make(cmd->pool, 1, sizeof(vfile_t)); // The DataFile is alternative with Redirect if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"))) && (NULL != (line = parse_sources(cmd, line, c->source)))) return line; - // The DataFile is alternative with Redirect + // Old style redirects go at the end if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "Redirect"))) && - (NULL != (line = parse_sources(cmd, line, c->source, 0)))) + (NULL != (line = parse_redirects(cmd, line, c->source)))) return line; line = apr_table_get(kvp, "RetryCount"); @@ -304,12 +212,7 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) // Index file can also be provided, there could be a default line = apr_table_get(kvp, "IndexFile"); - if (line) { - if (strlen(line) > 2 && line[0] == ':' && line[1] == '/') - c->idx.redirect = apr_pstrdup(cmd->pool, line + 1); - else - c->idx.fname = apr_pstrdup(cmd->pool, line); - } + c->idx.name = apr_pstrdup(cmd->pool, line); // Mime type is autodetected if not provided line = apr_table_get(kvp, "MimeType"); @@ -321,13 +224,13 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) if (line) c->skip_levels = atoi(line); - // If an emtpy tile is not provided, it falls through, which usually results in a 404 error - // If provided, it has an optional size and offset followed by file name which defaults to datafile - // read the empty tile + // If an emtpy tile is not provided, it falls through, which results in a 404 error + // If provided, it has an optional size and offset followed by file name which + // defaults to datafile read the empty tile // Default file name is the name of the first data file, if provided const char *datafname = NULL; for (int i = 0; i < c->source->nelts; i++) - if (NULL != (datafname = APR_ARRAY_IDX(c->source, i, source_t).fname)) + if (NULL != (datafname = APR_ARRAY_IDX(c->source, i, vfile_t).name)) break; const char *efname = datafname; @@ -335,7 +238,7 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) if (line) { char *last; // Try to read a figure first - c->empty.size = (int)apr_strtoi64(line, &last, 0); + c->empty.empty.size = (int)apr_strtoi64(line, &last, 0); // If that worked, try to get an offset too if (last != line) @@ -349,31 +252,32 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) // If we're provided a file name or a size, pre-read the empty tile in the if (efname && - (datafname == NULL || apr_strnatcmp(datafname, efname) || c->empty.size)) + (datafname == NULL || apr_strnatcmp(datafname, efname) || c->empty.empty.size)) { apr_file_t *efile; apr_off_t offset = c->eoffset; apr_status_t stat; // Use the temp pool for the file open, it will close it for us - if (!c->empty.size) { // Don't know the size, get it from the file + if (!c->empty.empty.size) { // Don't know the size, get it from the file apr_finfo_t finfo; stat = apr_stat(&finfo, efname, APR_FINFO_CSIZE, cmd->temp_pool); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't stat %s %pm", efname, &stat); - c->empty.size = (int)finfo.csize; + c->empty.empty.size = (int)finfo.csize; } stat = apr_file_open(&efile, efname, APR_FOPEN_READ | APR_FOPEN_BINARY, 0, cmd->temp_pool); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't open empty file %s, loaded from %s: %pm", efname, arg, &stat); - c->empty.buffer = (char *)apr_palloc(cmd->pool, static_cast(c->empty.size)); + c->empty.empty.buffer = reinterpret_cast(apr_palloc(cmd->pool, + static_cast(c->empty.empty.size))); stat = apr_file_seek(efile, APR_SET, &offset); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't seek empty tile %s: %pm", efname, &stat); - apr_size_t size = c->empty.size; - stat = apr_file_read(efile, (void *)c->empty.buffer, &size); + apr_size_t size = c->empty.empty.size; + stat = apr_file_read(efile, c->empty.empty.buffer, &size); if (APR_SUCCESS != stat) return apr_psprintf(cmd->pool, "Can't read from %s, loaded from %s: %pm", efname, arg, &stat); @@ -388,15 +292,15 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) tobase32(c->seed, c->eETag, 1); // Set the index file name based on the first data file, if there is only one - if (!c->idx.fname) { + if (!c->idx.name) { if (!datafname) return "Missing IndexFile or DataFile directive"; - c->idx.fname = apr_pstrdup(cmd->pool, datafname); + c->idx.name = apr_pstrdup(cmd->pool, datafname); char *last; - char *token = apr_strtok(c->idx.fname, ".", &last); // strtok destroys the idxfile + char *token = apr_strtok(c->idx.name, ".", &last); // strtok destroys the idxfile while (*last != 0 && token != NULL) token = apr_strtok(NULL, ".", &last); - memcpy(c->idx.fname, datafname, strlen(datafname)); // Get a new copy + memcpy(c->idx.name, datafname, strlen(datafname)); // Get a new copy if (token != NULL && strlen(token) == 3) memcpy(token, "idx", 3); } @@ -405,24 +309,6 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) return NULL; } -static int etag_matches(request_rec *r, const char *ETag) { - const char *ETagIn = apr_table_get(r->headers_in, "If-None-Match"); - return ETagIn != 0 && strstr(ETagIn, ETag); -} - -// Returns the empty tile if defined -static int send_empty_tile(request_rec *r) { - mrf_conf *cfg = get_conf(r); - if (etag_matches(r, cfg->eETag)) { - apr_table_setn(r->headers_out, "ETag", cfg->eETag); - return HTTP_NOT_MODIFIED; - } - - if (!cfg->empty.buffer) - return DECLINED; // Passthrough - return sendImage(r, cfg->empty); -} - // An open file handle and the matching file name, to be used as a note struct file_note { const char *name; @@ -435,13 +321,13 @@ static const apr_int32_t open_flags = APR_FOPEN_READ | APR_FOPEN_BINARY | APR_FO * Open or retrieve an connection cached file. */ static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, - const source_t &src, const char *note_name, apr_int32_t flags = open_flags) + const vfile_t &src, const char *note_name, apr_int32_t flags = open_flags) { apr_table_t *conn_notes = r->connection->notes; // Try to pick it up from the connection notes file_note *fn = (file_note *) apr_table_get(conn_notes, note_name); - if ((fn != NULL) && !apr_strnatcmp(src.fname, fn->name)) { // Match, set file and return + if ((fn != NULL) && !apr_strnatcmp(src.name, fn->name)) { // Match, set file and return *ppfh = fn->pfh; return APR_SUCCESS; } @@ -457,20 +343,20 @@ static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, fn = (file_note *)apr_palloc(pool, sizeof(file_note)); } - apr_status_t stat = apr_file_open(ppfh, src.fname, flags, 0, pool); + apr_status_t stat = apr_file_open(ppfh, src.name, flags, 0, pool); if (stat != APR_SUCCESS) return stat; // Fill the note and hook it up, then return fn->pfh = *ppfh; - fn->name = apr_pstrdup(pool, src.fname); // The old string will persist until cleaned by the pool + fn->name = apr_pstrdup(pool, src.name); apr_table_setn(conn_notes, note_name, (const char *) fn); return APR_SUCCESS; } // Open data file optimized for random access if possible static apr_status_t open_data_file(request_rec *r, apr_file_t **ppfh, - const source_t &src) + const vfile_t &src) { static const char data_note_name[] = "MRF_DATA_FILE"; @@ -506,22 +392,10 @@ static apr_status_t open_data_file(request_rec *r, apr_file_t **ppfh, return HTTP_INTERNAL_SERVER_ERROR; \ } -static bool our_request(request_rec *r, mrf_conf *cfg) { - if (r->method_number != M_GET || cfg->arr_rxp == NULL) - return false; - - char *url_to_match = r->args ? apr_pstrcat(r->pool, r->uri, "?", r->args, NULL) : r->uri; - for (int i = 0; i < cfg->arr_rxp->nelts; i++) { - ap_regex_t *m = APR_ARRAY_IDX(cfg->arr_rxp, i, ap_regex_t *); - if (!ap_regexec(m, url_to_match, 0, NULL, 0)) return true; // Found - } - return false; -} - // Return the first source which contains the index, adjusts the index offset if necessary -static const source_t *get_source(const apr_array_header_t *sources, range_t *index) { +static const vfile_t *get_source(const apr_array_header_t *sources, range_t *index) { for (int i = 0; i < sources->nelts; i++) { - source_t *source = &APR_ARRAY_IDX(sources, i, source_t); + vfile_t *source = &APR_ARRAY_IDX(sources, i, vfile_t); if ((source->range.offset == 0 && source->range.size == 0) || (index->offset >= source->range.offset && (source->range.size == 0 @@ -535,19 +409,19 @@ static const source_t *get_source(const apr_array_header_t *sources, range_t *in } static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { - mrf_conf *cfg = get_conf(r); + auto cfg = get_conf(r, &mrf_module); apr_file_t *idxf; if (open_connection_file(r, &idxf, cfg->idx, "MRF_INDEX_FILE")) return apr_psprintf(r->pool, - "Can't open index %s", cfg->idx.fname); + "Can't open index %s", cfg->idx.name); apr_size_t read_size = sizeof(range_t); if (apr_file_seek(idxf, APR_SET, &offset) || apr_file_read(idxf, idx, &read_size) || read_size != sizeof(range_t)) return apr_psprintf(r->pool, - "%s : Read error", cfg->idx.fname); + "%s : Read error", cfg->idx.name); idx->offset = be64toh(idx->offset); idx->size = be64toh(idx->size); @@ -560,8 +434,8 @@ static int handler(request_rec *r) if (r->args || r->method_number != M_GET) return DECLINED; - mrf_conf *cfg = get_conf(r); - if (!cfg->enabled || (cfg->indirect && !r->main) || !our_request(r, cfg)) + auto cfg = get_conf(r, &mrf_module); + if (!cfg->enabled || (cfg->indirect && !r->main) || !requestMatches(r, cfg->arr_rxp)) return DECLINED; apr_array_header_t *tokens = tokenize(r->pool, r->uri, '/'); @@ -585,7 +459,7 @@ static int handler(request_rec *r) // Don't allow access to levels less than zero, send the empty tile instead if (tile.l < 0) - return send_empty_tile(r); + return sendEmptyTile(r, cfg->empty); tile.l += cfg->skip_levels; // Check for bad requests, outside of the defined bounds @@ -604,11 +478,11 @@ static int handler(request_rec *r) // MRF index record is in network order if (index.size < 4) // Need at least four bytes for signature check - return send_empty_tile(r); + return sendEmptyTile(r, cfg->empty); if (MAX_TILE_SIZE < index.size) { // Tile is too large, log and send error code ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, "Tile too large in %s", - cfg->idx.fname); + cfg->idx.name); return HTTP_INTERNAL_SERVER_ERROR; } @@ -616,23 +490,26 @@ static int handler(request_rec *r) char ETag[16]; // Try to distribute the bits a bit to generate an ETag tobase32(cfg->seed ^ (index.size << 40) ^ index.offset, ETag); - if (etag_matches(r, ETag)) { + if (etagMatches(r, ETag)) { apr_table_set(r->headers_out, "ETag", ETag); return HTTP_NOT_MODIFIED; } // Now for the data part - const source_t *src = get_source(cfg->source, &index); - - if (!src || (!src->fname && !src->redirect)) + const vfile_t *src = get_source(cfg->source, &index); + const char *name = (src && src->name) ? src->name : nullptr; + if (!name) SERR_IF(true, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); + bool redirect = (strlen(name) > 2 && name[0] == ':' && name[1] == '/'); + apr_uint32_t *buffer = static_cast( apr_palloc(r->pool, static_cast(index.size))); SERR_IF(!buffer, "Memory allocation error in mod_mrf"); - if (src->redirect) { + if (redirect) { + const char *new_uri = name + 2; // Skip the :/ // TODO: S3 authorized requests ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); SERR_IF(!receive_filter, "Using redirect requires mod_receive"); @@ -651,7 +528,7 @@ static int handler(request_rec *r) int tries = cfg->tries; apr_time_t now = apr_time_now(); do { - request_rec *sr = ap_sub_req_lookup_uri(src->redirect, r, r->output_filters); + request_rec *sr = ap_sub_req_lookup_uri(new_uri, r, r->output_filters); apr_table_setn(sr->headers_in, "Range", Range); ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, sr, sr->connection); @@ -665,7 +542,7 @@ static int handler(request_rec *r) { // Abort here ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Can't fetch data from %s, took us %" APR_TIME_T_FMT, - src->redirect, apr_time_now() - now); + new_uri, apr_time_now() - now); return HTTP_SERVICE_UNAVAILABLE; } } while (rctx.size != static_cast(index.size)); @@ -674,15 +551,15 @@ static int handler(request_rec *r) { // Read from a local file apr_file_t *dataf; SERR_IF(open_data_file(r, &dataf, *src), - apr_psprintf(r->pool, "Can't open %s", src->fname)); + apr_psprintf(r->pool, "Can't open %s", name)); // We got the tile index, and is not empty SERR_IF(apr_file_seek(dataf, APR_SET, (apr_off_t *)&index.offset), - apr_psprintf(r->pool, "Seek error in %s", src->fname)); + apr_psprintf(r->pool, "Seek error in %s", name)); apr_size_t read_size = static_cast(index.size); SERR_IF(apr_file_read(dataf, buffer, &read_size) || read_size != index.size, - apr_psprintf(r->pool, "Can't read from %s", src->fname)); + apr_psprintf(r->pool, "Can't read from %s", name)); } // Looks fine, set the outgoing etag and then the image @@ -728,19 +605,8 @@ static const command_rec mrf_cmds[] = { NULL } }; - -// Return OK or DECLINED, anything else is error -//static int check_config(apr_pool_t *pconf, apr_pool_t *plog, apr_pool_t *ptemp, server_rec *server) -//{ -// return DECLINED; - // This gets called once for the whole server, it would have to check the configuration for every folder -//} - -static void mrf_register_hooks(apr_pool_t *p) - -{ +static void mrf_register_hooks(apr_pool_t *p) { ap_hook_handler(handler, NULL, NULL, APR_HOOK_FIRST); - // ap_hook_check_config(check_config, NULL, NULL, APR_HOOK_MIDDLE); } module AP_MODULE_DECLARE_DATA mrf_module = { From 58b4c67f32559e3486b6300ad2601f6b45d57f25 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sat, 20 Apr 2019 15:59:43 -0700 Subject: [PATCH 35/85] User the AHTSE TiledRaster --- src/mod_mrf.cpp | 169 +++++++----------------------------------------- 1 file changed, 22 insertions(+), 147 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 2df5a0e..9faec40 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -24,20 +24,10 @@ struct mrf_conf { apr_array_header_t *source; vfile_t idx; + TiledRaster raster; + // Forced mime-type, default is autodetected char *mime_type; - // Full raster size in pixels - sz size; - // Page size in pixels - sz pagesize; - - // Levels to skip at the top - int skip_levels; - int n_levels; - rset *rsets; - - empty_conf_t empty; - apr_off_t eoffset; // Turns the module functionality off int enabled; @@ -46,11 +36,6 @@ struct mrf_conf { // Used on remote data, how many times to try int tries; - - // ETag initializer - apr_uint64_t seed; - // Buffer for the emtpy tile etag - char eETag[16]; }; extern module AP_MODULE_DECLARE_DATA mrf_module; @@ -88,43 +73,10 @@ static char *get_xyzc_size(apr_pool_t *p, sz *size, return NULL; } -static void mrf_init(apr_pool_t *p, mrf_conf *c) { - rset level; - level.w = static_cast(1 + (c->size.x - 1) / c->pagesize.x); - level.h = static_cast(1 + (c->size.y - 1) / c->pagesize.y); - level.offset = 0; - // How many levels do we have - c->n_levels = 2 + ilogb(max(level.h, level.w) - 1); - c->rsets = (struct rset *)apr_pcalloc(p, sizeof(rset) * c->n_levels); - - // Populate rsets from the bottom, the way tile protcols count levels - // These are MRF rsets, not all of them are visible - struct rset *r = c->rsets + c->n_levels - 1; - for (int i = 0; i < c->n_levels; i++) { - *r-- = level; - // Prepare for the next level, assuming powers of two - // This is safe on all platforms that have large files (64bit signed offset) - // It will start failing if the file offset is larger than 63bits - // The c->size.z has to be first, to force the 64bit type - level.offset += c->size.z * level.w * level.h * sizeof(range_t); - level.w = 1 + (level.w - 1) / 2; - level.h = 1 + (level.h - 1) / 2; - } - // MRF has one tile at the top - ap_assert(c->rsets->h == 1 && c->rsets->w == 1); -} - -// Allow for one or more RegExp guard -// If present, at least one of them has to match the URL static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) { - // char *err_message = NULL; - if (c->arr_rxp == 0) - c->arr_rxp = apr_array_make(cmd->pool, 2, sizeof(ap_regex_t *)); - ap_regex_t **m = (ap_regex_t **)apr_array_push(c->arr_rxp); - *m = ap_pregcomp(cmd->pool, pattern, 0); - return (nullptr != *m) ? nullptr : "Bad regular expression"; + return add_regexp_to_array(cmd->pool, &c->arr_rxp, pattern); } // Parse a comma separated list of sources, add the entries to the array arr @@ -163,39 +115,18 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) mrf_conf *c = (mrf_conf *)dconf; const char *err_message; apr_table_t *kvp = readAHTSEConfig(cmd->temp_pool, arg, &err_message); - if (NULL == kvp) return err_message; + if (NULL == kvp) + return err_message; + + err_message = configRaster(cmd->pool, kvp, c->raster); + if (err_message) + return err_message; // Got the parsed kvp table, parse the configuration items const char *line; - char *err_prefix; - - line = apr_table_get(kvp, "Size"); - if (!line) - return apr_psprintf(cmd->temp_pool, "%s Size directive is mandatory", arg); - err_prefix = apr_psprintf(cmd->temp_pool, "%s Size", arg); - err_message = get_xyzc_size(cmd->temp_pool, &(c->size), line, err_prefix); - if (err_message) return err_message; - - // PageSize is optional, use reasonable defaults - c->pagesize.x = c->pagesize.z = 512; - c->pagesize.c = c->size.c; - c->pagesize.z = 1; - line = apr_table_get(kvp, "PageSize"); - if (line) { - err_prefix = apr_psprintf(cmd->temp_pool, "%s PageSize", arg); - err_message = get_xyzc_size(cmd->temp_pool, &(c->pagesize), line, err_prefix); - if (err_message) return err_message; - } - if (c->pagesize.c != c->size.c || c->pagesize.z != 1) - return apr_psprintf(cmd->temp_pool, "%s PageSize has invalid parameters", arg); + c->source = apr_array_make(cmd->pool, 1, sizeof(vfile_t)); - // Initialize the run-time structures - mrf_init(cmd->pool, c); - - if (!c->source) - c->source = apr_array_make(cmd->pool, 1, sizeof(vfile_t)); - - // The DataFile is alternative with Redirect + // The DataFile, multiple times, includes redirects if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"))) && (NULL != (line = parse_sources(cmd, line, c->source)))) return line; @@ -219,11 +150,6 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) if (line) c->mime_type = apr_pstrdup(cmd->pool, line); - // Skip levels, from the top of the MRF - line = apr_table_get(kvp, "SkippedLevels"); - if (line) - c->skip_levels = atoi(line); - // If an emtpy tile is not provided, it falls through, which results in a 404 error // If provided, it has an optional size and offset followed by file name which // defaults to datafile read the empty tile @@ -235,61 +161,8 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) const char *efname = datafname; line = apr_table_get(kvp, "EmptyTile"); - if (line) { - char *last; - // Try to read a figure first - c->empty.empty.size = (int)apr_strtoi64(line, &last, 0); - - // If that worked, try to get an offset too - if (last != line) - apr_strtoff(&(c->eoffset), last, &last, 0); - - // If there is anything left - while (*last && isspace(*last)) last++; - if (*last != 0) - efname = last; - } - - // If we're provided a file name or a size, pre-read the empty tile in the - if (efname && - (datafname == NULL || apr_strnatcmp(datafname, efname) || c->empty.empty.size)) - { - apr_file_t *efile; - apr_off_t offset = c->eoffset; - apr_status_t stat; - - // Use the temp pool for the file open, it will close it for us - if (!c->empty.empty.size) { // Don't know the size, get it from the file - apr_finfo_t finfo; - stat = apr_stat(&finfo, efname, APR_FINFO_CSIZE, cmd->temp_pool); - if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't stat %s %pm", efname, &stat); - c->empty.empty.size = (int)finfo.csize; - } - - stat = apr_file_open(&efile, efname, APR_FOPEN_READ | APR_FOPEN_BINARY, 0, cmd->temp_pool); - if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't open empty file %s, loaded from %s: %pm", - efname, arg, &stat); - c->empty.empty.buffer = reinterpret_cast(apr_palloc(cmd->pool, - static_cast(c->empty.empty.size))); - stat = apr_file_seek(efile, APR_SET, &offset); - if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't seek empty tile %s: %pm", efname, &stat); - apr_size_t size = c->empty.empty.size; - stat = apr_file_read(efile, c->empty.empty.buffer, &size); - if (APR_SUCCESS != stat) - return apr_psprintf(cmd->pool, "Can't read from %s, loaded from %s: %pm", - efname, arg, &stat); - apr_file_close(efile); - } - - line = apr_table_get(kvp, "ETagSeed"); - // Ignore the flag - int flag; - c->seed = line ? base32decode(line, &flag) : 0; - // Set the missing tile etag, with the extra bit set - tobase32(c->seed, c->eETag, 1); + if (line && strlen(line) && (err_message = readFile(cmd->pool, c->raster.missing.empty, line))) + return err_message; // Set the index file name based on the first data file, if there is only one if (!c->idx.name) { @@ -452,19 +325,21 @@ static int handler(request_rec *r) tile.y = apr_atoi64(*(char **)apr_array_pop(tokens)); REQ_ERR_IF(errno); tile.l = apr_atoi64(*(char **)apr_array_pop(tokens)); REQ_ERR_IF(errno); + const TiledRaster &raster(cfg->raster); + // We can ignore the error on this one, defaults to zero // The parameter before the level can't start with a digit for an extra-dimensional MRF - if (cfg->size.z != 1 && tokens->nelts) + if (raster.size.z != 1 && tokens->nelts) tile.z = apr_atoi64(*(char **)apr_array_pop(tokens)); // Don't allow access to levels less than zero, send the empty tile instead if (tile.l < 0) - return sendEmptyTile(r, cfg->empty); + return sendEmptyTile(r, raster.missing); - tile.l += cfg->skip_levels; + tile.l += raster.skip; // Check for bad requests, outside of the defined bounds - REQ_ERR_IF(tile.l >= cfg->n_levels); - rset *level = cfg->rsets + tile.l; + REQ_ERR_IF(tile.l >= raster.n_levels); + rset *level = raster.rsets + tile.l; REQ_ERR_IF(tile.x >= level->w || tile.y >= level->h); // Offset of the index entry for this tile @@ -478,7 +353,7 @@ static int handler(request_rec *r) // MRF index record is in network order if (index.size < 4) // Need at least four bytes for signature check - return sendEmptyTile(r, cfg->empty); + return sendEmptyTile(r, raster.missing); if (MAX_TILE_SIZE < index.size) { // Tile is too large, log and send error code ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, "Tile too large in %s", @@ -489,7 +364,7 @@ static int handler(request_rec *r) // Check for conditional ETag here, no need to get the data char ETag[16]; // Try to distribute the bits a bit to generate an ETag - tobase32(cfg->seed ^ (index.size << 40) ^ index.offset, ETag); + tobase32(raster.seed ^ (index.size << 40) ^ index.offset, ETag); if (etagMatches(r, ETag)) { apr_table_set(r->headers_out, "ETag", ETag); return HTTP_NOT_MODIFIED; From 14ad82caddb236934bafebfb4e8b8f12456858ec Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sat, 20 Apr 2019 16:01:21 -0700 Subject: [PATCH 36/85] Forced mime type no longer supported --- README.md | 3 --- src/mod_mrf.cpp | 8 -------- 2 files changed, 11 deletions(-) diff --git a/README.md b/README.md index fd6bf70..cdce382 100644 --- a/README.md +++ b/README.md @@ -55,9 +55,6 @@ AHTSE Control Directives for this module are: Otherwise it uses the first data file name with the extension changed to .idx It can be a redirect path in the host namespace, if it starts with a colon : -***MimeType string*** - - Optional, defaults to autodetect. - ***EmptyTile Size Offset FileName*** - Optional, provides the tile content to be sent when the requested tile is missing. By default the request is ignored, which results in a 404 error if a fallback mechanism does not exist. if present, the first number is assumed to be the size, second is offset. If filename is not given, the first data file name is used. diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 9faec40..5e89727 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -26,9 +26,6 @@ struct mrf_conf { TiledRaster raster; - // Forced mime-type, default is autodetected - char *mime_type; - // Turns the module functionality off int enabled; // If set, only secondary requests are allowed @@ -145,11 +142,6 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) line = apr_table_get(kvp, "IndexFile"); c->idx.name = apr_pstrdup(cmd->pool, line); - // Mime type is autodetected if not provided - line = apr_table_get(kvp, "MimeType"); - if (line) - c->mime_type = apr_pstrdup(cmd->pool, line); - // If an emtpy tile is not provided, it falls through, which results in a 404 error // If provided, it has an optional size and offset followed by file name which // defaults to datafile read the empty tile From f7b3ef5b25dc660a13ee5c84b4c532d61e4d14c6 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 15:29:37 -0700 Subject: [PATCH 37/85] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e3c334..5875860 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # mod_mrf [AHTSE](https://github.com/lucianpls/AHTSE) An apache module that serves tiles directly from a local MRF, 2D or 3D. -This module takes two apache configuration directives: +Apache configuration directives: **MRF On|Off** From d5ef141da1a00df2076ef1307d226733725dad24 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 15:29:51 -0700 Subject: [PATCH 38/85] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5875860..acd2add 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # mod_mrf [AHTSE](https://github.com/lucianpls/AHTSE) -An apache module that serves tiles directly from a local MRF, 2D or 3D. +An apache module that serves tiles directly from a local MRF, 2D or 3D. Apache configuration directives: **MRF On|Off** From 07897465cf7058f528e66e1770f621be48104745 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 15:38:31 -0700 Subject: [PATCH 39/85] Removed unused function --- src/mod_mrf.cpp | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 5e89727..f0c4f31 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -15,7 +15,6 @@ #define CMD_FUNC (cmd_func) using namespace std; - NS_AHTSE_USE struct mrf_conf { @@ -49,27 +48,6 @@ static inline void *create_dir_config(apr_pool_t *p, char *dummy) return c; } -// Returns NULL if it worked as expected, returns a four integer value from "x y", "x y z" or "x y z c" -static char *get_xyzc_size(apr_pool_t *p, sz *size, - const char *value, const char*err_prefix) -{ - char *s; - if (!value) - return apr_psprintf(p, "%s directive missing", err_prefix); - size->x = apr_strtoi64(value, &s, 0); - size->y = apr_strtoi64(s, &s, 0); - size->c = 3; - size->z = 1; - if (errno == 0 && *s) { // Read optional third and fourth integers - size->z = apr_strtoi64(s, &s, 0); - if (*s) - size->c = apr_strtoi64(s, &s, 0); - } // Raster size is 4 params max - if (errno || *s) - return apr_psprintf(p, "%s incorrect", err_prefix); - return NULL; -} - static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, const char *pattern) { From 36b0253f2a258e028fe72cef0e96ac566871baa8 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 15:50:55 -0700 Subject: [PATCH 40/85] removed mrf directive Presence of mrf regexp is sufficient --- README.md | 53 +++++++++++++++++++++++++++---------------------- src/mod_mrf.cpp | 31 +++++++++++------------------ 2 files changed, 41 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index cdce382..1df05ed 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,9 @@ An apache module that serves tiles directly from a local MRF, 2D or 3D. This module takes two apache configuration directives: -**MRF On|Off** - - Defaults to on if the MRF_ConfigurationFile is provided - **MRF_RegExp** - Required, only requests matching this pattern are handled. It can appear multiple times + If not provided, the module is inactive **MRF_Indirect On|Off** @@ -26,46 +22,55 @@ AHTSE Control Directives for this module are: ***DataFile path start_offset size*** - The path to the MRF data file to serve tiles from. Start and size are optional, by default a single DataFile is used. At least one DataFile directive is required. If the path start - with colon (:) followed by slash /, the path is interpreted as an internal redirect to a path - within the same server, starting from DocRoot. Otherwise it is assumed to be a local file + with ://, the path is interpreted as an internal redirect to a path + within the same server, starting from DocRoot. Otherwise it is assumed to be a local file name. May appear multiple times, with different start offset and size values. If the values are - present, read operations within start_offset and start_offset + size are made to the data file, after the read - offset is adjusted downward by start_offset. - If the read offset falls outside the range, the other DataFile entries are searched, in the order in which they - appear in the configuration file, but first all local files will be checked and then the redirects. This allows an MRF data file to be split into multiple parts. Single tiles - cannot be split, but overlapping ranges are allowed, the first match will be used. Only one read - operation will be issued, to the first DataFile entry that matches the range. If the read fails, + present, read operations within start_offset and start_offset + size are made to the data file, + after the read offset is adjusted downward by start_offset. + If the read offset falls outside the range, the other DataFile entries are searched, + in the order in which they appear in the configuration file. Old style redirects are tested last. + The multiple entries allows an MRF data file to be split into multiple parts. Single tiles + cannot be split between sources, but overlapping ranges between source are allowed. Only one read + operation is issued, to the first DataFile entry that matches the range. If the read fails, the server will report an error. - Start offset and size default to zero. Zero size means that any read above the offset will be done in - this data file. + Start offset and size default to zero. Zero size means that any read above the offset will be + considered present in this data file. ***Size X Y Z C*** - - Mandatory, at least x and y, the size in pixels of the input MRF. Z defaults to 1 and C defaults to 3 (these are usually not meaningful) + - Mandatory, at least x and y, the size in pixels of the input MRF. + Z defaults to 1 and C defaults to 3 (these are usually not meaningful) ***PageSize X Y 1 C*** - - Optional, the pagesize in pixels. X and Y default to 512. Z has to be 1 if C is provided, which has to match the C value from size + - Optional, the pagesize in pixels. X and Y default to 512. + Z has to be 1 if C is provided, which has to match the C value from size ***RetryCount N*** - - Optional, [0 - 99). If the DataFiles are redirects, how many times to attempt retrieving data from - the source path. Defaults to 4, which means it will try 5 times. + - Optional, [0 - 99). If the DataFiles are redirects, how many times to retry a redirected + read that fails to retun the requested data. The Default is 5. ***IndexFile string*** - Optional, the index file name. Can only be provided once. If not provided it uses the data file name if its extension is not three letters. Otherwise it uses the first data file name with the extension changed to .idx - It can be a redirect path in the host namespace, if it starts with a colon : + It can be a redirect path in the host namespace, if it starts with :// ***EmptyTile Size Offset FileName*** - - Optional, provides the tile content to be sent when the requested tile is missing. By default the request is ignored, which results in a 404 error if a fallback mechanism does not exist. if present, the first number is assumed to be the size, second is offset. If filename is not given, the first data file name is used. + - Optional, provides the tile content to be sent when the requested tile is missing. + By default the request is ignored, which results in a 404 error if a fallback mechanism does not + exist. If present, the first number is assumed to be the size, second is offset. If filename is + not given, the first data file name is used. ***SkippedLevels N*** - - Optional, how many levels to ignore, at the top of the MRF pyramid. For example a GCS pyramid will have to skip the one tile level, so this should be 1 + - Optional, how many levels to ignore, at the top of the MRF pyramid. For example a GCS pyramid + will have to skip the one tile level, so this should be 1 ***ETagSeed base32_string*** - - Optional, 64 bits as 13 base32 digits [0-9a-v], defaults to 0. The empty tile ETag will be this value but 65th bit is set, also the only value that has this bit set. All the other tiles have 64 bit ETags that depend on this value. + - Optional, 64 bits as 13 base32 digits [0-9a-v], defaults to 0. The empty tile ETag will be + this value but 65th bit is set, also the only value that has this bit set. All the other tiles + have 64 bit ETags that depend on this value. ***Redirect path start_offset size*** - *Deprecated*, use the DataFile directive and start path with : + *Deprecated*, use the DataFile directive and start path with :// For better performance on local files, the httpd source against which this module is compiled should include support for random file access optimization. A patch file for libapr is provided, see apr_FOPEN_RANDOM.patch diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index f0c4f31..fa3746e 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -20,18 +20,19 @@ NS_AHTSE_USE struct mrf_conf { // array of guard regexp, one of them has to match apr_array_header_t *arr_rxp; + TiledRaster raster; + + // At least one source, but there could be more apr_array_header_t *source; + + // The MRF index file vfile_t idx; - TiledRaster raster; + // Used for redirect, how many times to try + int tries; - // Turns the module functionality off - int enabled; // If set, only secondary requests are allowed int indirect; - - // Used on remote data, how many times to try - int tries; }; extern module AP_MODULE_DECLARE_DATA mrf_module; @@ -148,7 +149,6 @@ static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) memcpy(token, "idx", 3); } - c->enabled = 1; return NULL; } @@ -278,7 +278,7 @@ static int handler(request_rec *r) return DECLINED; auto cfg = get_conf(r, &mrf_module); - if (!cfg->enabled || (cfg->indirect && !r->main) || !requestMatches(r, cfg->arr_rxp)) + if ((cfg->indirect && !r->main) || !requestMatches(r, cfg->arr_rxp)) return DECLINED; apr_array_header_t *tokens = tokenize(r->pool, r->uri, '/'); @@ -346,13 +346,14 @@ static int handler(request_rec *r) if (!name) SERR_IF(true, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); - bool redirect = (strlen(name) > 2 && name[0] == ':' && name[1] == '/'); - apr_uint32_t *buffer = static_cast( apr_palloc(r->pool, static_cast(index.size))); SERR_IF(!buffer, "Memory allocation error in mod_mrf"); + // A name starting with :// is a redirect. The last / will be preserved, so the + // redirect path is absolute within this host + bool redirect = (strlen(name) > 3 && name[0] == ':' && name[1] == '/' && name[2] == '/'); if (redirect) { const char *new_uri = name + 2; // Skip the :/ // TODO: S3 authorized requests @@ -386,7 +387,7 @@ static int handler(request_rec *r) && (0 == tries--)) { // Abort here ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, - "Can't fetch data from %s, took us %" APR_TIME_T_FMT, + "Can't fetch data from %s, took %" APR_TIME_T_FMT "us", new_uri, apr_time_now() - now); return HTTP_SERVICE_UNAVAILABLE; } @@ -415,14 +416,6 @@ static int handler(request_rec *r) static const command_rec mrf_cmds[] = { - AP_INIT_FLAG( - "MRF", - CMD_FUNC ap_set_flag_slot, - (void *)APR_OFFSETOF(mrf_conf, enabled), - ACCESS_CONF, - "mod_mrf enable, defaults to on if configuration is provided" - ), - AP_INIT_FLAG( "MRF_Indirect", CMD_FUNC ap_set_flag_slot, From 1d80f4a4821aa91f282272849de06e441bbb302d Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 16:01:26 -0700 Subject: [PATCH 41/85] No need for mrf prefix for local functions --- src/mod_mrf.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index fa3746e..a34fef0 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -85,7 +85,7 @@ static const char *parse_redirects(cmd_parms *cmd, const char *src, return parse_sources(cmd, src, arr, true); } -static const char *mrf_file_set(cmd_parms *cmd, void *dconf, const char *arg) +static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) { ap_assert(sizeof(apr_off_t) == 8); mrf_conf *c = (mrf_conf *)dconf; @@ -414,7 +414,7 @@ static int handler(request_rec *r) return sendImage(r, temp); } -static const command_rec mrf_cmds[] = +static const command_rec cmds[] = { AP_INIT_FLAG( "MRF_Indirect", @@ -434,7 +434,7 @@ static const command_rec mrf_cmds[] = AP_INIT_TAKE1( "MRF_ConfigurationFile", - CMD_FUNC mrf_file_set, // Callback + CMD_FUNC file_set, // Callback 0, // Self-pass argument ACCESS_CONF, // availability "The configuration file for this module" @@ -443,7 +443,7 @@ static const command_rec mrf_cmds[] = { NULL } }; -static void mrf_register_hooks(apr_pool_t *p) { +static void register_hooks(apr_pool_t *p) { ap_hook_handler(handler, NULL, NULL, APR_HOOK_FIRST); } @@ -453,6 +453,6 @@ module AP_MODULE_DECLARE_DATA mrf_module = { 0, // No dir_merge 0, // No server_config 0, // No server_merge - mrf_cmds, // configuration directives - mrf_register_hooks // processing hooks + cmds, // configuration directives + register_hooks // processing hooks }; From 92451057b907224d1bca060258b3732565747bf4 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 16:08:59 -0700 Subject: [PATCH 42/85] Treat the old Redirect directive correctly --- src/mod_mrf.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index a34fef0..1bffdf7 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -67,9 +67,15 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, char *input = APR_ARRAY_IDX(inputs, i, char *); char *fname = ap_getword_white_nc(arr->pool, &input); - if (!fname) + if (!fname || strlen(fname) < 1) return "Missing source name"; + if (redir) { // Check that it is absolute and add :/ + if (fname[0] != '/') + return "Only absolute paths are allowed for Redirect"; + fname = apr_pstrcat(arr->pool, ":/", fname, NULL); + } + entry->name = fname; // See if there are more arguments, should be offset and size From 3e32ef92c84a9924b9e2c032c9d95c77e968d621 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 16:11:33 -0700 Subject: [PATCH 43/85] better error message --- src/mod_mrf.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 1bffdf7..677f7ef 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -72,7 +72,8 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, if (redir) { // Check that it is absolute and add :/ if (fname[0] != '/') - return "Only absolute paths are allowed for Redirect"; + return apr_pstrcat(cmd->pool, "Only absolute redirects as allowed, ", + fname, " is not absolute", NULL); fname = apr_pstrcat(arr->pool, ":/", fname, NULL); } From c682da6ada8906cc0aaa0114c5066751ca1e67ae Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 17:09:26 -0700 Subject: [PATCH 44/85] wrote vfile_pread Virtualized file pread, handles redirects or local files --- src/mod_mrf.cpp | 101 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 7 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 677f7ef..3475c7a 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -122,7 +122,7 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) line = apr_table_get(kvp, "RetryCount"); c->tries = 1 + (line ? atoi(line) : 0); if ((c->tries < 1) || (c->tries > 100)) - return "Invalid RetryCount value, should be 0 to 99, defaults to 4"; + return "Invalid RetryCount value, should be 0 to 99"; // Index file can also be provided, there could be a default line = apr_table_get(kvp, "IndexFile"); @@ -161,7 +161,7 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) // An open file handle and the matching file name, to be used as a note struct file_note { - const char *name; + const char *key; apr_file_t *pfh; }; @@ -177,7 +177,7 @@ static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, // Try to pick it up from the connection notes file_note *fn = (file_note *) apr_table_get(conn_notes, note_name); - if ((fn != NULL) && !apr_strnatcmp(src.name, fn->name)) { // Match, set file and return + if ((fn != NULL) && !apr_strnatcmp(src.name, fn->key)) { // Match, set file and return *ppfh = fn->pfh; return APR_SUCCESS; } @@ -194,12 +194,12 @@ static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, } apr_status_t stat = apr_file_open(ppfh, src.name, flags, 0, pool); - if (stat != APR_SUCCESS) + if (stat != APR_SUCCESS) return stat; // Fill the note and hook it up, then return fn->pfh = *ppfh; - fn->name = apr_pstrdup(pool, src.name); + fn->key = apr_pstrdup(pool, src.name); apr_table_setn(conn_notes, note_name, (const char *) fn); return APR_SUCCESS; } @@ -243,7 +243,7 @@ static apr_status_t open_data_file(request_rec *r, apr_file_t **ppfh, } // Return the first source which contains the index, adjusts the index offset if necessary -static const vfile_t *get_source(const apr_array_header_t *sources, range_t *index) { +static const vfile_t *pick_source(const apr_array_header_t *sources, range_t *index) { for (int i = 0; i < sources->nelts; i++) { vfile_t *source = &APR_ARRAY_IDX(sources, i, vfile_t); if ((source->range.offset == 0 && source->range.size == 0) @@ -258,6 +258,93 @@ static const vfile_t *get_source(const apr_array_header_t *sources, range_t *ind return NULL; } +// Like pread, except not really thread safe +static int vfile_pread(request_rec *r, void *ptr, int size, off_t offset, const vfile_t *fh) { + auto cfg = get_conf(r, &mrf_module); + const char *name = fh->name; + + bool redirect = (strlen(name) > 3 && name[0] == ':' && name[1] == '/'); + + if (redirect) { // Remote file, just use a range request + // TODO: S3 authorized requests + ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); + if (!receive_filter) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Can't find receive filter, did you load mod_receive?"); + return 0; + } + + name = fh->name + 2; // Skip the :/ + + // Get a buffer for the received image + receive_ctx rctx; + rctx.buffer = reinterpret_cast(ptr); + rctx.maxsize = size; + rctx.size = 0; + + // Data file is on a remote site a range request redirect with a range header + char *Range = apr_psprintf(r->pool, + "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT, + offset, offset + size); + + // S3 may return less than requested, so we retry the request a couple of times + int tries = cfg->tries; + bool failed = false; + apr_time_t now = apr_time_now(); + do { + request_rec *sr = ap_sub_req_lookup_uri(name, r, r->output_filters); + apr_table_setn(sr->headers_in, "Range", Range); + ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, + sr, sr->connection); + int status = ap_run_sub_req(sr); + ap_remove_output_filter(rf); + ap_destroy_sub_req(sr); + + if ((status != APR_SUCCESS || sr->status != HTTP_PARTIAL_CONTENT + || rctx.size != static_cast(size)) + && (0 == tries--)) + { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Can't fetch data from %s, took %" APR_TIME_T_FMT "us", + name, apr_time_now() - now); + failed = true; + } + } while (!failed && rctx.size != static_cast(size)); + + return rctx.size; + } + + // Local file, open, seek, read, close + apr_pool_t *pool = r->pool; + apr_file_t *pfh; + apr_status_t stat = apr_file_open(&pfh, name, open_flags, 0, pool); + if (stat != APR_SUCCESS) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Can't open file %s", name); + return 0; // No read + } + + apr_size_t sz = size; + try { + stat = apr_file_seek(pfh, APR_SET, (apr_off_t *)&offset); + if (stat != APR_SUCCESS) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Seek error in %s offset %" APR_OFF_T_FMT, name, offset); + sz = 0; + throw 0; // No read + } + + sz = size; + stat = apr_file_read(pfh, ptr, &sz); + } + catch (int &e) { + apr_file_close(pfh); // Close the file + } + + // return whatever was read + return sz; +} + static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { auto cfg = get_conf(r, &mrf_module); apr_file_t *idxf; @@ -348,7 +435,7 @@ static int handler(request_rec *r) } // Now for the data part - const vfile_t *src = get_source(cfg->source, &index); + const vfile_t *src = pick_source(cfg->source, &index); const char *name = (src && src->name) ? src->name : nullptr; if (!name) SERR_IF(true, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); From b26394a6d48cc1db74a635ceb258542e5bb9b16d Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 17:26:26 -0700 Subject: [PATCH 45/85] Use vfile_pread --- src/mod_mrf.cpp | 80 +++++++------------------------------------------ 1 file changed, 10 insertions(+), 70 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 3475c7a..3040313 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -259,7 +259,7 @@ static const vfile_t *pick_source(const apr_array_header_t *sources, range_t *in } // Like pread, except not really thread safe -static int vfile_pread(request_rec *r, void *ptr, int size, off_t offset, const vfile_t *fh) { +static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, const vfile_t *fh) { auto cfg = get_conf(r, &mrf_module); const char *name = fh->name; @@ -345,24 +345,17 @@ static int vfile_pread(request_rec *r, void *ptr, int size, off_t offset, const return sz; } +// Indirect read of index static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { auto cfg = get_conf(r, &mrf_module); - apr_file_t *idxf; + static int size = sizeof(range_t); - if (open_connection_file(r, &idxf, cfg->idx, "MRF_INDEX_FILE")) - return apr_psprintf(r->pool, - "Can't open index %s", cfg->idx.name); - - apr_size_t read_size = sizeof(range_t); - if (apr_file_seek(idxf, APR_SET, &offset) - || apr_file_read(idxf, idx, &read_size) - || read_size != sizeof(range_t)) - return apr_psprintf(r->pool, - "%s : Read error", cfg->idx.name); + if (size != vfile_pread(r, idx, size, offset, &cfg->idx)) + return "Read error"; idx->offset = be64toh(idx->offset); idx->size = be64toh(idx->size); - return nullptr; + return nullptr; // Success } static int handler(request_rec *r) @@ -444,63 +437,10 @@ static int handler(request_rec *r) apr_palloc(r->pool, static_cast(index.size))); SERR_IF(!buffer, "Memory allocation error in mod_mrf"); - - // A name starting with :// is a redirect. The last / will be preserved, so the - // redirect path is absolute within this host - bool redirect = (strlen(name) > 3 && name[0] == ':' && name[1] == '/' && name[2] == '/'); - if (redirect) { - const char *new_uri = name + 2; // Skip the :/ - // TODO: S3 authorized requests - ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); - SERR_IF(!receive_filter, "Using redirect requires mod_receive"); - - // Get a buffer for the received image - receive_ctx rctx; - rctx.buffer = reinterpret_cast(buffer); - rctx.maxsize = static_cast(index.size); - rctx.size = 0; - - // Data file is on a remote site a range request redirect with a range header - char *Range = apr_psprintf(r->pool, "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT, - index.offset, index.offset + index.size); - - // S3 may return less than requested, so we retry the request a couple of times - int tries = cfg->tries; - apr_time_t now = apr_time_now(); - do { - request_rec *sr = ap_sub_req_lookup_uri(new_uri, r, r->output_filters); - apr_table_setn(sr->headers_in, "Range", Range); - ap_filter_t *rf = ap_add_output_filter_handle(receive_filter, &rctx, - sr, sr->connection); - int status = ap_run_sub_req(sr); - ap_remove_output_filter(rf); - ap_destroy_sub_req(sr); - - if ((status != APR_SUCCESS || sr->status != HTTP_PARTIAL_CONTENT - || rctx.size != static_cast(index.size)) - && (0 == tries--)) - { // Abort here - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, - "Can't fetch data from %s, took %" APR_TIME_T_FMT "us", - new_uri, apr_time_now() - now); - return HTTP_SERVICE_UNAVAILABLE; - } - } while (rctx.size != static_cast(index.size)); - } - else - { // Read from a local file - apr_file_t *dataf; - SERR_IF(open_data_file(r, &dataf, *src), - apr_psprintf(r->pool, "Can't open %s", name)); - - // We got the tile index, and is not empty - SERR_IF(apr_file_seek(dataf, APR_SET, (apr_off_t *)&index.offset), - apr_psprintf(r->pool, "Seek error in %s", name)); - - apr_size_t read_size = static_cast(index.size); - SERR_IF(apr_file_read(dataf, buffer, &read_size) || read_size != index.size, - apr_psprintf(r->pool, "Can't read from %s", name)); - } + + int size = static_cast(index.size); + SERR_IF(size != vfile_pread(r, buffer, size, index.offset, src), + "Data read error"); // Looks fine, set the outgoing etag and then the image apr_table_set(r->headers_out, "ETag", ETag); From 8793c3743a0b92807d053fa0de28f41dccd3fb51 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 17:28:37 -0700 Subject: [PATCH 46/85] Removed unused code Functionality should move to the vfile functions --- src/mod_mrf.cpp | 64 ------------------------------------------------- 1 file changed, 64 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 3040313..e1d51b2 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -167,70 +167,6 @@ struct file_note { static const apr_int32_t open_flags = APR_FOPEN_READ | APR_FOPEN_BINARY | APR_FOPEN_LARGEFILE; -/* - * Open or retrieve an connection cached file. - */ -static apr_status_t open_connection_file(request_rec *r, apr_file_t **ppfh, - const vfile_t &src, const char *note_name, apr_int32_t flags = open_flags) -{ - apr_table_t *conn_notes = r->connection->notes; - - // Try to pick it up from the connection notes - file_note *fn = (file_note *) apr_table_get(conn_notes, note_name); - if ((fn != NULL) && !apr_strnatcmp(src.name, fn->key)) { // Match, set file and return - *ppfh = fn->pfh; - return APR_SUCCESS; - } - - // Use the connection pool for the note and file, to ensure it gets closed with the connection - apr_pool_t *pool = r->connection->pool; - - if (fn != NULL) { // We have a file note but it is not the right file - apr_table_unset(conn_notes, note_name); // Unhook the existing note - apr_file_close(fn->pfh); // Close the existing file - } - else { // no previous note, allocate a clean one - fn = (file_note *)apr_palloc(pool, sizeof(file_note)); - } - - apr_status_t stat = apr_file_open(ppfh, src.name, flags, 0, pool); - if (stat != APR_SUCCESS) - return stat; - - // Fill the note and hook it up, then return - fn->pfh = *ppfh; - fn->key = apr_pstrdup(pool, src.name); - apr_table_setn(conn_notes, note_name, (const char *) fn); - return APR_SUCCESS; -} - -// Open data file optimized for random access if possible -static apr_status_t open_data_file(request_rec *r, apr_file_t **ppfh, - const vfile_t &src) -{ - static const char data_note_name[] = "MRF_DATA_FILE"; - -#if defined(APR_FOPEN_RANDOM) - // apr has portable support for random access to files - return open_connection_file(r, ppfh, src, open_flags | APR_FOPEN_RANDOM, - data_note_name); -#else - - apr_status_t stat = - open_connection_file(r, ppfh, src, data_note_name, open_flags); - -#if !defined(POSIX_FADV_RANDOM) - return stat; - -#else // last chance, turn random flag on if supported - apr_os_file_t fd; - if (APR_SUCCESS == apr_os_file_get(&fd, *ppfh)) - posix_fadvise(static_cast(fd), 0, 0, POSIX_FADV_RANDOM); - return stat; -#endif -#endif // APR_FOPEN_RANDOM -} - // Quiet error #define REQ_ERR_IF(X) if (X) {\ return HTTP_BAD_REQUEST; \ From a19cc7c6f62dbfed70f186f11c22cee359011f93 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 18:09:13 -0700 Subject: [PATCH 47/85] small cleanup --- src/mod_mrf.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index e1d51b2..31467fe 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -262,7 +262,7 @@ static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, co apr_size_t sz = size; try { - stat = apr_file_seek(pfh, APR_SET, (apr_off_t *)&offset); + stat = apr_file_seek(pfh, APR_SET, &offset); if (stat != APR_SUCCESS) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Seek error in %s offset %" APR_OFF_T_FMT, name, offset); @@ -274,11 +274,12 @@ static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, co stat = apr_file_read(pfh, ptr, &sz); } catch (int &e) { + sz = e; apr_file_close(pfh); // Close the file } // return whatever was read - return sz; + return static_cast(sz); } // Indirect read of index @@ -369,19 +370,17 @@ static int handler(request_rec *r) if (!name) SERR_IF(true, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); - apr_uint32_t *buffer = static_cast( - apr_palloc(r->pool, static_cast(index.size))); - SERR_IF(!buffer, + apr_size_t size = static_cast(index.size); + storage_manager img(apr_palloc(r->pool, size), size); + + SERR_IF(img.buffer, "Memory allocation error in mod_mrf"); - - int size = static_cast(index.size); - SERR_IF(size != vfile_pread(r, buffer, size, index.offset, src), + SERR_IF(img.size != vfile_pread(r, img.buffer, img.size, index.offset, src), "Data read error"); // Looks fine, set the outgoing etag and then the image apr_table_set(r->headers_out, "ETag", ETag); - storage_manager temp = { (char *)buffer, static_cast(index.size) }; - return sendImage(r, temp); + return sendImage(r, img); } static const command_rec cmds[] = From c58be51d2fa4e8fdea913de77d663f43634fab3f Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 18:23:33 -0700 Subject: [PATCH 48/85] cleanup --- src/mod_mrf.cpp | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 31467fe..4e6da2d 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -20,16 +20,19 @@ NS_AHTSE_USE struct mrf_conf { // array of guard regexp, one of them has to match apr_array_header_t *arr_rxp; + + // The raster represented by this MRF configuration TiledRaster raster; // At least one source, but there could be more apr_array_header_t *source; - // The MRF index file + // The MRF index file, required vfile_t idx; // Used for redirect, how many times to try - int tries; + // defaults to 5 + int retries; // If set, only secondary requests are allowed int indirect; @@ -41,11 +44,11 @@ extern module AP_MODULE_DECLARE_DATA mrf_module; APLOG_USE_MODULE(mrf); #endif -static inline void *create_dir_config(apr_pool_t *p, char *dummy) +static void *create_dir_config(apr_pool_t *p, char *dummy) { mrf_conf *c = (mrf_conf *)apr_pcalloc(p, sizeof(mrf_conf)); - c->tries = 5; + c->retries = 5; return c; } @@ -86,11 +89,7 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, return nullptr; } -static const char *parse_redirects(cmd_parms *cmd, const char *src, - apr_array_header_t *arr) -{ - return parse_sources(cmd, src, arr, true); -} +#define parse_redirects(cmd, src, arr) parse_sources(cmd, src, arr, true) static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) { @@ -120,8 +119,8 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) return line; line = apr_table_get(kvp, "RetryCount"); - c->tries = 1 + (line ? atoi(line) : 0); - if ((c->tries < 1) || (c->tries > 100)) + c->retries = 1 + (line ? atoi(line) : 0); + if ((c->retries < 1) || (c->retries > 100)) return "Invalid RetryCount value, should be 0 to 99"; // Index file can also be provided, there could be a default @@ -201,8 +200,13 @@ static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, co bool redirect = (strlen(name) > 3 && name[0] == ':' && name[1] == '/'); - if (redirect) { // Remote file, just use a range request + if (redirect) { + // Remote file, just use a range request // TODO: S3 authorized requests + + // Skip the ":/" used to mark a redirect + name = fh->name + 2; + ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); if (!receive_filter) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, @@ -210,8 +214,6 @@ static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, co return 0; } - name = fh->name + 2; // Skip the :/ - // Get a buffer for the received image receive_ctx rctx; rctx.buffer = reinterpret_cast(ptr); @@ -224,7 +226,7 @@ static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, co offset, offset + size); // S3 may return less than requested, so we retry the request a couple of times - int tries = cfg->tries; + int tries = cfg->retries; bool failed = false; apr_time_t now = apr_time_now(); do { @@ -282,7 +284,7 @@ static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, co return static_cast(sz); } -// Indirect read of index +// Indirect read of index, returns error message or null static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { auto cfg = get_conf(r, &mrf_module); static int size = sizeof(range_t); @@ -292,12 +294,10 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { idx->offset = be64toh(idx->offset); idx->size = be64toh(idx->size); - return nullptr; // Success + return nullptr; } -static int handler(request_rec *r) -{ - // Only get and no arguments +static int handler(request_rec *r) { if (r->args || r->method_number != M_GET) return DECLINED; @@ -383,8 +383,7 @@ static int handler(request_rec *r) return sendImage(r, img); } -static const command_rec cmds[] = -{ +static const command_rec cmds[] = { AP_INIT_FLAG( "MRF_Indirect", CMD_FUNC ap_set_flag_slot, From f7e020327c9f24b16f1c636343b1d6872627680b Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 18:35:33 -0700 Subject: [PATCH 49/85] configuration cleanup --- src/mod_mrf.cpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 4e6da2d..679b1d8 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -105,10 +105,17 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) return err_message; // Got the parsed kvp table, parse the configuration items - const char *line; + // Usually there is a single source, but we still need an array c->source = apr_array_make(cmd->pool, 1, sizeof(vfile_t)); - // The DataFile, multiple times, includes redirects + const char *line; + + // Index file can also be provided, there could be a default + line = apr_table_get(kvp, "IndexFile"); + c->idx.name = apr_pstrdup(cmd->pool, line); + + // The DataFile, required, multiple times, includes redirects + line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"); if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"))) && (NULL != (line = parse_sources(cmd, line, c->source)))) return line; @@ -118,39 +125,32 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) (NULL != (line = parse_redirects(cmd, line, c->source)))) return line; + // Check that we have at least one data file + const char *firstname = nullptr; + if (0 == c->source->nelts || (firstname = APR_ARRAY_IDX(c->source, 0, vfile_t).name)) + return "Need at least one DataFile directive"; + line = apr_table_get(kvp, "RetryCount"); c->retries = 1 + (line ? atoi(line) : 0); if ((c->retries < 1) || (c->retries > 100)) return "Invalid RetryCount value, should be 0 to 99"; - // Index file can also be provided, there could be a default - line = apr_table_get(kvp, "IndexFile"); - c->idx.name = apr_pstrdup(cmd->pool, line); - // If an emtpy tile is not provided, it falls through, which results in a 404 error // If provided, it has an optional size and offset followed by file name which // defaults to datafile read the empty tile // Default file name is the name of the first data file, if provided - const char *datafname = NULL; - for (int i = 0; i < c->source->nelts; i++) - if (NULL != (datafname = APR_ARRAY_IDX(c->source, i, vfile_t).name)) - break; - - const char *efname = datafname; line = apr_table_get(kvp, "EmptyTile"); if (line && strlen(line) && (err_message = readFile(cmd->pool, c->raster.missing.empty, line))) - return err_message; + return err_message; // Set the index file name based on the first data file, if there is only one if (!c->idx.name) { - if (!datafname) - return "Missing IndexFile or DataFile directive"; - c->idx.name = apr_pstrdup(cmd->pool, datafname); + c->idx.name = apr_pstrdup(cmd->pool, firstname); char *last; char *token = apr_strtok(c->idx.name, ".", &last); // strtok destroys the idxfile while (*last != 0 && token != NULL) token = apr_strtok(NULL, ".", &last); - memcpy(c->idx.name, datafname, strlen(datafname)); // Get a new copy + memcpy(c->idx.name, firstname, strlen(firstname)); // Get a new copy if (token != NULL && strlen(token) == 3) memcpy(token, "idx", 3); } From 389f02daaf329cd99840fad75e061af606ab8147 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 21 Apr 2019 18:40:33 -0700 Subject: [PATCH 50/85] Update mod_mrf.cpp --- src/mod_mrf.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 679b1d8..3987175 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -44,15 +44,15 @@ extern module AP_MODULE_DECLARE_DATA mrf_module; APLOG_USE_MODULE(mrf); #endif -static void *create_dir_config(apr_pool_t *p, char *dummy) -{ +static void *create_dir_config(apr_pool_t *p, char *dummy) { mrf_conf *c = - (mrf_conf *)apr_pcalloc(p, sizeof(mrf_conf)); + reinterpret_cast(apr_pcalloc(p, sizeof(mrf_conf))); c->retries = 5; return c; } -static const char *set_regexp(cmd_parms *cmd, mrf_conf *c, +static const char *set_regexp(cmd_parms *cmd, + mrf_conf *c, const char *pattern) { return add_regexp_to_array(cmd->pool, &c->arr_rxp, pattern); @@ -71,7 +71,7 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, char *fname = ap_getword_white_nc(arr->pool, &input); if (!fname || strlen(fname) < 1) - return "Missing source name"; + return "Source name missing"; if (redir) { // Check that it is absolute and add :/ if (fname[0] != '/') @@ -117,17 +117,17 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) // The DataFile, required, multiple times, includes redirects line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"); if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "DataFile"))) && - (NULL != (line = parse_sources(cmd, line, c->source)))) - return line; + (NULL != (err_message = parse_sources(cmd, line, c->source)))) + return err_message; // Old style redirects go at the end if ((NULL != (line = apr_table_getm(cmd->temp_pool, kvp, "Redirect"))) && - (NULL != (line = parse_redirects(cmd, line, c->source)))) - return line; + (NULL != (err_message = parse_redirects(cmd, line, c->source)))) + return err_message; // Check that we have at least one data file const char *firstname = nullptr; - if (0 == c->source->nelts || (firstname = APR_ARRAY_IDX(c->source, 0, vfile_t).name)) + if (0 == c->source->nelts || !(firstname = APR_ARRAY_IDX(c->source, 0, vfile_t).name)) return "Need at least one DataFile directive"; line = apr_table_get(kvp, "RetryCount"); From 796804b087ddc9079b4308e806c5d59b73737a66 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 22 Apr 2019 13:54:41 -0700 Subject: [PATCH 51/85] debugged Runs, no file handle caching, both local and remote files --- src/mod_mrf.cpp | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 3987175..ea83d59 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -45,8 +45,8 @@ APLOG_USE_MODULE(mrf); #endif static void *create_dir_config(apr_pool_t *p, char *dummy) { - mrf_conf *c = - reinterpret_cast(apr_pcalloc(p, sizeof(mrf_conf))); + mrf_conf *c = reinterpret_cast( + apr_pcalloc(p, sizeof(mrf_conf))); c->retries = 5; return c; } @@ -58,7 +58,7 @@ static const char *set_regexp(cmd_parms *cmd, return add_regexp_to_array(cmd->pool, &c->arr_rxp, pattern); } -// Parse a comma separated list of sources, add the entries to the array arr +// Parse a comma separated list of sources, add the entries to the array // Source may include offset and size, white space separated static const char *parse_sources(cmd_parms *cmd, const char *src, apr_array_header_t *arr, bool redir = false) @@ -95,7 +95,8 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) { ap_assert(sizeof(apr_off_t) == 8); mrf_conf *c = (mrf_conf *)dconf; - const char *err_message; + const char *err_message, *line; + apr_table_t *kvp = readAHTSEConfig(cmd->temp_pool, arg, &err_message); if (NULL == kvp) return err_message; @@ -108,8 +109,6 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) // Usually there is a single source, but we still need an array c->source = apr_array_make(cmd->pool, 1, sizeof(vfile_t)); - const char *line; - // Index file can also be provided, there could be a default line = apr_table_get(kvp, "IndexFile"); c->idx.name = apr_pstrdup(cmd->pool, line); @@ -166,17 +165,6 @@ struct file_note { static const apr_int32_t open_flags = APR_FOPEN_READ | APR_FOPEN_BINARY | APR_FOPEN_LARGEFILE; -// Quiet error -#define REQ_ERR_IF(X) if (X) {\ - return HTTP_BAD_REQUEST; \ -} - -// Logged error -#define SERR_IF(X, msg) if (X) { \ - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "%s", msg);\ - return HTTP_INTERNAL_SERVER_ERROR; \ -} - // Return the first source which contains the index, adjusts the index offset if necessary static const vfile_t *pick_source(const apr_array_header_t *sources, range_t *index) { for (int i = 0; i < sources->nelts; i++) { @@ -297,6 +285,17 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { return nullptr; } +// Quiet error +#define REQ_ERR_IF(X) if (X) {\ + return HTTP_BAD_REQUEST; \ +} + +// Logged error +#define SERR_IF(X, msg) if (X) { \ + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "%s", msg);\ + return HTTP_INTERNAL_SERVER_ERROR; \ +} + static int handler(request_rec *r) { if (r->args || r->method_number != M_GET) return DECLINED; @@ -337,8 +336,8 @@ static int handler(request_rec *r) { REQ_ERR_IF(tile.x >= level->w || tile.y >= level->h); // Offset of the index entry for this tile - apr_off_t tidx_offset = level->offset + - sizeof(range_t) * (tile.x + level->w * (tile.z * level->h + tile.y)); + apr_off_t tidx_offset = sizeof(range_t) * (level->tiles + + + level->w * (tile.z * level->h + tile.y) + tile.x); range_t index; const char *message; @@ -373,7 +372,7 @@ static int handler(request_rec *r) { apr_size_t size = static_cast(index.size); storage_manager img(apr_palloc(r->pool, size), size); - SERR_IF(img.buffer, + SERR_IF(!img.buffer, "Memory allocation error in mod_mrf"); SERR_IF(img.size != vfile_pread(r, img.buffer, img.size, index.offset, src), "Data read error"); From 41ffc37815c223f6f2f1bb4b625c144a286e7fea Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 22 Apr 2019 14:36:09 -0700 Subject: [PATCH 52/85] match libahtse variable rename --- src/mod_mrf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index ea83d59..56fd556 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -139,7 +139,7 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) // defaults to datafile read the empty tile // Default file name is the name of the first data file, if provided line = apr_table_get(kvp, "EmptyTile"); - if (line && strlen(line) && (err_message = readFile(cmd->pool, c->raster.missing.empty, line))) + if (line && strlen(line) && (err_message = readFile(cmd->pool, c->raster.missing.data, line))) return err_message; // Set the index file name based on the first data file, if there is only one From 8d77b8332351039b3917ca81e54185dc6446847d Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 22 Apr 2019 16:43:21 -0700 Subject: [PATCH 53/85] file handle reuse per connection --- src/mod_mrf.cpp | 127 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 87 insertions(+), 40 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 56fd556..3167746 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -36,6 +36,9 @@ struct mrf_conf { // If set, only secondary requests are allowed int indirect; + + // If set, file handles are not held open + int dynamic; }; extern module AP_MODULE_DECLARE_DATA mrf_module; @@ -154,15 +157,13 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) memcpy(token, "idx", 3); } + line = apr_table_get(kvp, "Dynamic"); + if (line) + c->dynamic = true; + return NULL; } -// An open file handle and the matching file name, to be used as a note -struct file_note { - const char *key; - apr_file_t *pfh; -}; - static const apr_int32_t open_flags = APR_FOPEN_READ | APR_FOPEN_BINARY | APR_FOPEN_LARGEFILE; // Return the first source which contains the index, adjusts the index offset if necessary @@ -181,8 +182,56 @@ static const vfile_t *pick_source(const apr_array_header_t *sources, range_t *in return NULL; } +// An open file handle and the matching file name, to be used as a note +struct file_note { + const vfile_t *fh; + apr_file_t *pfh; +}; + +// +// Open or retrieve a connection cached file handle +// Do no close the returned file handle, it will be removed when the connection drops +// +// Only one opened handle exists per connection/token pair +// This may lead to less caching, but avoids having too many opened files +// +static apr_status_t openConnFile(request_rec *r, apr_file_t **ppfh, const vfile_t *fh, + const char *token, apr_int32_t extra_open_flags = 0) +{ + apr_table_t *conn_notes = r->connection->notes; + + file_note *fn = (file_note *)apr_table_get(conn_notes, token); + if (fn && fn->fh == fh) { // Match, return the handle + *ppfh = fn->pfh; + return APR_SUCCESS; + } + + // Use the connection pool, it will close the file when it gets dropped + apr_pool_t *pool = r->connection->pool; + if (!fn) { // new connection file + fn = reinterpret_cast(apr_pcalloc(pool, sizeof(file_note))); + } + else { // Not the right file, clean it up + apr_table_unset(conn_notes, token); // Does not remove the storage + apr_file_close(fn->pfh); + } + + apr_status_t stat = apr_file_open(ppfh, fh->name, open_flags || extra_open_flags, 0, pool); + if (APR_SUCCESS != stat) + return stat; + + // Update the note and hook it up before returning + fn->fh = fh; + fn->pfh = *ppfh; + apr_table_setn(conn_notes, token, (const char *)fn); + return APR_SUCCESS; +} + // Like pread, except not really thread safe -static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, const vfile_t *fh) { +// The token is used for connection caching +static int vfile_pread(request_rec *r, storage_manager &mgr, + apr_off_t offset, const vfile_t *fh, const char *token = "MRF_DATA") +{ auto cfg = get_conf(r, &mrf_module); const char *name = fh->name; @@ -204,14 +253,14 @@ static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, co // Get a buffer for the received image receive_ctx rctx; - rctx.buffer = reinterpret_cast(ptr); - rctx.maxsize = size; + rctx.buffer = mgr.buffer; + rctx.maxsize = mgr.size; rctx.size = 0; // Data file is on a remote site a range request redirect with a range header char *Range = apr_psprintf(r->pool, "bytes=%" APR_UINT64_T_FMT "-%" APR_UINT64_T_FMT, - offset, offset + size); + offset, offset + mgr.size); // S3 may return less than requested, so we retry the request a couple of times int tries = cfg->retries; @@ -226,58 +275,56 @@ static int vfile_pread(request_rec *r, void *ptr, int size, apr_off_t offset, co ap_remove_output_filter(rf); ap_destroy_sub_req(sr); - if ((status != APR_SUCCESS || sr->status != HTTP_PARTIAL_CONTENT - || rctx.size != static_cast(size)) - && (0 == tries--)) + if ((status != APR_SUCCESS + || sr->status != HTTP_PARTIAL_CONTENT + || rctx.size != mgr.size) + && (0 == tries--)) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Can't fetch data from %s, took %" APR_TIME_T_FMT "us", name, apr_time_now() - now); failed = true; } - } while (!failed && rctx.size != static_cast(size)); + } while (!failed && rctx.size != mgr.size); return rctx.size; - } + } // Redirect read - // Local file, open, seek, read, close - apr_pool_t *pool = r->pool; + // Local file apr_file_t *pfh; - apr_status_t stat = apr_file_open(&pfh, name, open_flags, 0, pool); + apr_status_t stat; + + // Don't keep handles open + stat = cfg->dynamic ? + apr_file_open(&pfh, name, open_flags, 0, r->pool) + : openConnFile(r, &pfh, fh, token, APR_FOPEN_BUFFERED); + if (stat != APR_SUCCESS) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Can't open file %s", name); - return 0; // No read + return 0; // No file } - apr_size_t sz = size; - try { - stat = apr_file_seek(pfh, APR_SET, &offset); - if (stat != APR_SUCCESS) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, - "Seek error in %s offset %" APR_OFF_T_FMT, name, offset); - sz = 0; - throw 0; // No read - } - - sz = size; - stat = apr_file_read(pfh, ptr, &sz); - } - catch (int &e) { - sz = e; - apr_file_close(pfh); // Close the file + apr_size_t sz = static_cast(mgr.size); + stat = apr_file_seek(pfh, APR_SET, &offset); + if (APR_SUCCESS != stat || APR_SUCCESS != apr_file_read(pfh, mgr.buffer, &sz)) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Read error in %s offset %" APR_OFF_T_FMT, name, offset); + sz = 0; } - // return whatever was read - return static_cast(sz); + if (cfg->dynamic) + apr_file_close(pfh); + + return (mgr.size = static_cast(sz)); } // Indirect read of index, returns error message or null static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { auto cfg = get_conf(r, &mrf_module); - static int size = sizeof(range_t); + storage_manager dst(idx, sizeof(range_t)); - if (size != vfile_pread(r, idx, size, offset, &cfg->idx)) + if (sizeof(range_t) != vfile_pread(r, dst, offset, &cfg->idx, "MRF_INDEX")) return "Read error"; idx->offset = be64toh(idx->offset); @@ -374,7 +421,7 @@ static int handler(request_rec *r) { SERR_IF(!img.buffer, "Memory allocation error in mod_mrf"); - SERR_IF(img.size != vfile_pread(r, img.buffer, img.size, index.offset, src), + SERR_IF(img.size != vfile_pread(r, img, index.offset, src), "Data read error"); // Looks fine, set the outgoing etag and then the image From 14a8fa6fc2038e16186c165819f0267d97c8f2f8 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 22 Apr 2019 17:03:11 -0700 Subject: [PATCH 54/85] Document Dynamic directive --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1df05ed..c085670 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,8 @@ AHTSE Control Directives for this module are: It can be a redirect path in the host namespace, if it starts with :// ***EmptyTile Size Offset FileName*** - - Optional, provides the tile content to be sent when the requested tile is missing. + - Optional, provides the tile content to be sent when the requested tile is missing. + The file has to be local, since the empty tile is read at start-up By default the request is ignored, which results in a 404 error if a fallback mechanism does not exist. If present, the first number is assumed to be the size, second is offset. If filename is not given, the first data file name is used. @@ -69,6 +70,11 @@ AHTSE Control Directives for this module are: this value but 65th bit is set, also the only value that has this bit set. All the other tiles have 64 bit ETags that depend on this value. +***Dynamic On*** + - Optional, flags the local files as dynamic, disabling any caching or file handle reuse. To be used + when the MRF files are changed at run-time, avoiding stale or even broken content. MRF in-place + modification do not require this flag because the old content is still available. + ***Redirect path start_offset size*** *Deprecated*, use the DataFile directive and start path with :// From 000d49f0332eb1d49f537c62dbd7f57cfc8c08e4 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 23 Apr 2019 13:18:27 -0700 Subject: [PATCH 55/85] Finally, canned index functionality --- README.md | 7 ++++- src/mod_mrf.cpp | 82 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c085670..55c3ff2 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,12 @@ AHTSE Control Directives for this module are: ***Dynamic On*** - Optional, flags the local files as dynamic, disabling any caching or file handle reuse. To be used when the MRF files are changed at run-time, avoiding stale or even broken content. MRF in-place - modification do not require this flag because the old content is still available. + modification do not require this flag because the old content is still available + +***CannedIndex On*** + - Optional, flags the index file as a canned format index, see mrf_apps/can.cpp. This is a dense + format index that can be much smaller. Should be used only when needed, and not recommended when + Dynamic is also on ***Redirect path start_offset size*** *Deprecated*, use the DataFile directive and start path with :// diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 3167746..055eec9 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -17,6 +17,37 @@ using namespace std; NS_AHTSE_USE +// Max count, should never happen +#define NPOS 0xffffffff +// Block size for canned format +#define BSZ 512 + +// Bit set count for 32bit values +#if defined(_WIN32) +#include +#define bsc __popcnt +#else +#define bsc __builtint_popcount +#endif + +// The next two functions are from the mrf/mrf_apps/can program +inline bool is_on(uint32_t *values, int bit) { + return 0 != (values[1 + bit / 32] & (static_cast(1) << bit % 32)); +} + +static inline uint64_t hsize(uint64_t in_size) { + return 16 + 16 * ((96 * BSZ - 1 + in_size) / (96 * BSZ)); +} + +inline uint32_t block_count(uint32_t *values, int bit) { + if (!is_on(values, bit)) + return NPOS; + return (values[0] + + bsc(values[1]) * (((bit / 32) & 1) | (bit / 64)) + + bsc(values[2]) * (bit / 64) + + bsc(values[1 + (bit / 32)] & ((1ULL << (bit % 32)) - 1))); +} + struct mrf_conf { // array of guard regexp, one of them has to match apr_array_header_t *arr_rxp; @@ -39,6 +70,9 @@ struct mrf_conf { // If set, file handles are not held open int dynamic; + + // the canned index header size, or 0 for normal index + uint64_t can_hsize; }; extern module AP_MODULE_DECLARE_DATA mrf_module; @@ -157,10 +191,15 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) memcpy(token, "idx", 3); } - line = apr_table_get(kvp, "Dynamic"); - if (line) + if ((line = apr_table_get(kvp, "Dynamic")) && getBool(line)) c->dynamic = true; + // The original index file size is the number of tiles * 16 + // Since the MRF always ends with a single tile, the total number + // of tiles in the MRF is equal to the number of tiles at level 0 + size.z + if ((line = apr_table_get(kvp, "CannedIndex")) && getBool(line)) + c->can_hsize = hsize((c->raster.size.z + c->raster.rsets[0].tiles) * 16); + return NULL; } @@ -295,9 +334,10 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, apr_status_t stat; // Don't keep handles open - stat = cfg->dynamic ? - apr_file_open(&pfh, name, open_flags, 0, r->pool) - : openConnFile(r, &pfh, fh, token, APR_FOPEN_BUFFERED); + if (cfg->dynamic) + stat = apr_file_open(&pfh, name, open_flags, 0, r->pool); + else + stat = openConnFile(r, &pfh, fh, token, APR_FOPEN_BUFFERED); if (stat != APR_SUCCESS) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, @@ -315,6 +355,7 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, if (cfg->dynamic) apr_file_close(pfh); + // Don't close non-dynamic mode handles, they are reused return (mgr.size = static_cast(sz)); } @@ -324,6 +365,34 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { auto cfg = get_conf(r, &mrf_module); storage_manager dst(idx, sizeof(range_t)); + if (cfg->can_hsize) { // No checks that the file is correct + // Original block offset + uint64_t boffset = offset / BSZ; + + // Read the line containing the target bit + uint32_t line[4]; + storage_manager lmgr(line, 16); + // DEBUG ONLY + off_t loffset = 16 * (1 + (boffset / 96)); + + if (16 != vfile_pread(r, lmgr, 16 * (1 + (boffset / 96)), &cfg->idx, "MRF_INDEX")) + return "Bitmap read error"; + + // Change to host endian + for (int i = 0; i < 4; i++) + line[i] = be32toh(line[i]); + + // The relocated block number for the original index record + uint64_t blockn = block_count(line, static_cast(boffset % 96)); + if (NPOS == blockn) { + idx->size = idx->offset = 0; + return nullptr; + } + + // Adjust the offset before reading the data + offset = cfg->can_hsize + blockn * BSZ + offset % BSZ; + } + if (sizeof(range_t) != vfile_pread(r, dst, offset, &cfg->idx, "MRF_INDEX")) return "Read error"; @@ -419,8 +488,7 @@ static int handler(request_rec *r) { apr_size_t size = static_cast(index.size); storage_manager img(apr_palloc(r->pool, size), size); - SERR_IF(!img.buffer, - "Memory allocation error in mod_mrf"); + SERR_IF(!img.buffer, "Memory allocation error in mod_mrf"); SERR_IF(img.size != vfile_pread(r, img, index.offset, src), "Data read error"); From 509b4d770c4672c644170e9f0d91a68b61a61492 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 23 Apr 2019 13:28:24 -0700 Subject: [PATCH 56/85] linux fixes --- src/mod_mrf.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 055eec9..33433da 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -27,7 +27,8 @@ NS_AHTSE_USE #include #define bsc __popcnt #else -#define bsc __builtint_popcount +// This only works in gcc +#define bsc __builtin_popcount #endif // The next two functions are from the mrf/mrf_apps/can program @@ -372,10 +373,8 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { // Read the line containing the target bit uint32_t line[4]; storage_manager lmgr(line, 16); - // DEBUG ONLY - off_t loffset = 16 * (1 + (boffset / 96)); - - if (16 != vfile_pread(r, lmgr, 16 * (1 + (boffset / 96)), &cfg->idx, "MRF_INDEX")) + if (16 != vfile_pread(r, lmgr, + 16 * (1 + (boffset / 96)), &cfg->idx, "MRF_INDEX")) return "Bitmap read error"; // Change to host endian From 3a07e085d04dbcd7726a166d2a523226d4435d0d Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 23 Apr 2019 13:37:00 -0700 Subject: [PATCH 57/85] Add AllowParams HTTP configuration directive --- src/mod_mrf.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 33433da..e50d4f6 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -72,6 +72,9 @@ struct mrf_conf { // If set, file handles are not held open int dynamic; + // Ignore URL parameters + int allowparams; + // the canned index header size, or 0 for normal index uint64_t can_hsize; }; @@ -412,11 +415,13 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { } static int handler(request_rec *r) { - if (r->args || r->method_number != M_GET) + if (r->method_number != M_GET) return DECLINED; auto cfg = get_conf(r, &mrf_module); - if ((cfg->indirect && !r->main) || !requestMatches(r, cfg->arr_rxp)) + if ((r->args && ! cfg->allowparams) || + (cfg->indirect && !r->main) || + !requestMatches(r, cfg->arr_rxp)) return DECLINED; apr_array_header_t *tokens = tokenize(r->pool, r->uri, '/'); @@ -521,6 +526,14 @@ static const command_rec cmds[] = { "The configuration file for this module" ), + AP_INIT_FLAG( + "AllowParams", + CMD_FUNC ap_set_flag_slot, + (void *)APR_OFFSETOF(mrf_conf, allowparams), + ACCESS_CONF, + "If set, this configuration only responds to subrequests" + ), + { NULL } }; From 96d748cf27da91da2c753353f1eac3be558901fc Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 24 Apr 2019 15:14:07 -0700 Subject: [PATCH 58/85] Change and documented allow params directive --- README.md | 4 ++++ src/mod_mrf.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 55c3ff2..53c32c0 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,10 @@ This module takes two apache configuration directives: - Empty lines, lines that start with # are considered comments - Unknown directives are ignored + **MRF_AllowParams On** + + If set, the MRF module allows get requests with parameters. Default is Off + AHTSE Control Directives for this module are: ***DataFile path start_offset size*** diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index e50d4f6..889537f 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -527,7 +527,7 @@ static const command_rec cmds[] = { ), AP_INIT_FLAG( - "AllowParams", + "MRF_AllowParams", CMD_FUNC ap_set_flag_slot, (void *)APR_OFFSETOF(mrf_conf, allowparams), ACCESS_CONF, From e311535a63c5d6abd46762fcc10a120d4f56a0bb Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Fri, 10 May 2019 16:56:57 -0700 Subject: [PATCH 59/85] minor tweaks --- mod_mrf.vcxproj | 2 +- src/mod_mrf.cpp | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index 279961d..7de8a0f 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -113,7 +113,7 @@ $(OutDir)$(TargetName)$(TargetExt) - copy /y $(OutDir)$(TargetName)$(TargetExt) \Apache24\modules + copy /y $(TargetPath) \Apache24\modules diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 889537f..7fea703 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -12,8 +12,6 @@ #include #include -#define CMD_FUNC (cmd_func) - using namespace std; NS_AHTSE_USE @@ -180,7 +178,8 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) // defaults to datafile read the empty tile // Default file name is the name of the first data file, if provided line = apr_table_get(kvp, "EmptyTile"); - if (line && strlen(line) && (err_message = readFile(cmd->pool, c->raster.missing.data, line))) + if (line && strlen(line) && (err_message = readFile( + cmd->pool, c->raster.missing.data, line))) return err_message; // Set the index file name based on the first data file, if there is only one @@ -349,7 +348,7 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, return 0; // No file } - apr_size_t sz = static_cast(mgr.size); + apr_size_t sz = static_cast(mgr.size); stat = apr_file_seek(pfh, APR_SET, &offset); if (APR_SUCCESS != stat || APR_SUCCESS != apr_file_read(pfh, mgr.buffer, &sz)) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, @@ -477,7 +476,7 @@ static int handler(request_rec *r) { // Check for conditional ETag here, no need to get the data char ETag[16]; // Try to distribute the bits a bit to generate an ETag - tobase32(raster.seed ^ (index.size << 40) ^ index.offset, ETag); + tobase32((raster.seed ^ (index.size << 40)) ^ index.offset, ETag); if (etagMatches(r, ETag)) { apr_table_set(r->headers_out, "ETag", ETag); return HTTP_NOT_MODIFIED; @@ -486,8 +485,7 @@ static int handler(request_rec *r) { // Now for the data part const vfile_t *src = pick_source(cfg->source, &index); const char *name = (src && src->name) ? src->name : nullptr; - if (!name) - SERR_IF(true, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); + SERR_IF(!name, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); apr_size_t size = static_cast(index.size); storage_manager img(apr_palloc(r->pool, size), size); @@ -504,7 +502,7 @@ static int handler(request_rec *r) { static const command_rec cmds[] = { AP_INIT_FLAG( "MRF_Indirect", - CMD_FUNC ap_set_flag_slot, + (cmd_func) ap_set_flag_slot, (void *)APR_OFFSETOF(mrf_conf, indirect), ACCESS_CONF, "If set, this configuration only responds to subrequests" @@ -512,7 +510,7 @@ static const command_rec cmds[] = { AP_INIT_TAKE1( "MRF_RegExp", - (cmd_func)set_regexp, + (cmd_func) set_regexp, 0, // Self-pass argument ACCESS_CONF, // availability "Regular expression that the URL has to match. At least one is required." @@ -520,7 +518,7 @@ static const command_rec cmds[] = { AP_INIT_TAKE1( "MRF_ConfigurationFile", - CMD_FUNC file_set, // Callback + (cmd_func) file_set, // Callback 0, // Self-pass argument ACCESS_CONF, // availability "The configuration file for this module" @@ -528,7 +526,7 @@ static const command_rec cmds[] = { AP_INIT_FLAG( "MRF_AllowParams", - CMD_FUNC ap_set_flag_slot, + (cmd_func) ap_set_flag_slot, (void *)APR_OFFSETOF(mrf_conf, allowparams), ACCESS_CONF, "If set, this configuration only responds to subrequests" From 6cef4646141902fac4691929ef46a0f7a93ec086 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 25 Jul 2019 11:48:11 -0700 Subject: [PATCH 60/85] Allow parameters by default --- src/mod_mrf.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 7fea703..6e03280 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -70,9 +70,6 @@ struct mrf_conf { // If set, file handles are not held open int dynamic; - // Ignore URL parameters - int allowparams; - // the canned index header size, or 0 for normal index uint64_t can_hsize; }; @@ -418,8 +415,7 @@ static int handler(request_rec *r) { return DECLINED; auto cfg = get_conf(r, &mrf_module); - if ((r->args && ! cfg->allowparams) || - (cfg->indirect && !r->main) || + if ((cfg->indirect && !r->main) || !requestMatches(r, cfg->arr_rxp)) return DECLINED; @@ -524,14 +520,6 @@ static const command_rec cmds[] = { "The configuration file for this module" ), - AP_INIT_FLAG( - "MRF_AllowParams", - (cmd_func) ap_set_flag_slot, - (void *)APR_OFFSETOF(mrf_conf, allowparams), - ACCESS_CONF, - "If set, this configuration only responds to subrequests" - ), - { NULL } }; From caf382fbc8c2470999b142c236a9a2bcf0f34313 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 14 Oct 2019 15:46:13 -0700 Subject: [PATCH 61/85] Send missing tile on the negative side of area --- src/mod_mrf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 6e03280..961fdec 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -441,7 +441,7 @@ static int handler(request_rec *r) { tile.z = apr_atoi64(*(char **)apr_array_pop(tokens)); // Don't allow access to levels less than zero, send the empty tile instead - if (tile.l < 0) + if (tile.l < 0 || tile.x < 0 || tile.y < 0) return sendEmptyTile(r, raster.missing); tile.l += raster.skip; From 49a67cbab1d87bfab6d22c8e88c1a6e08204150f Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 23 Oct 2019 17:08:12 -0700 Subject: [PATCH 62/85] Load module a bit later --- src/mod_mrf.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 961fdec..0fe2ded 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -524,7 +524,8 @@ static const command_rec cmds[] = { }; static void register_hooks(apr_pool_t *p) { - ap_hook_handler(handler, NULL, NULL, APR_HOOK_FIRST); + // Up in the stack, but leave APR_HOOK_FIRST available + ap_hook_handler(handler, NULL, NULL, APR_HOOK_FIRST + 1); } module AP_MODULE_DECLARE_DATA mrf_module = { From da903e15fe1caaf853b35ac47ee5603697563da9 Mon Sep 17 00:00:00 2001 From: lplesea Date: Thu, 24 Oct 2019 19:38:15 +0000 Subject: [PATCH 63/85] More portable Makefile --- src/Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Makefile b/src/Makefile index 0507bfe..3bb082f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,4 +1,5 @@ -C_SRC = mod_mrf.cpp +TGT = mod_mrf +C_SRC = $(TGT).cpp FILES = $(C_SRC) DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE $(DEBUG) @@ -9,13 +10,13 @@ DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE $(DEBUG) # LIBTOOL, DEBUG ... include Makefile.lcl -TARGET = .libs/mod_mrf.so +TARGET = .libs/$(TGT).so # Can't use apxs to build c++ modules # The options used here might depend on how apache was built $(TARGET) : $(FILES) - $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(INCLUDES) -pthread -c -o mod_mrf.lo $(C_SRC) && touch mod_mrf.slo - $(LIBTOOL) --silent --mode=link g++ -o mod_mrf.la -rpath $(DEST) -module -avoid-version mod_mrf.lo + $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(INCLUDES) -pthread -c -o $(TGT).lo $(C_SRC) && touch $(TGT).slo + $(LIBTOOL) --silent --mode=link g++ -o $(TGT).la -rpath $(DEST) -module -avoid-version $(TGT).lo install : $(TARGET) $(SUDO) cp $(TARGET) $(DEST) From e40562ec65e683f17dbe7bc6fdac303a8dd47abb Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 24 Oct 2019 15:35:08 -0700 Subject: [PATCH 64/85] swap only if needed --- src/mod_mrf.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 0fe2ded..f4c4d3e 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -361,6 +361,7 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, } // Indirect read of index, returns error message or null +// MRF index file is network order static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { auto cfg = get_conf(r, &mrf_module); storage_manager dst(idx, sizeof(range_t)); @@ -376,9 +377,11 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { 16 * (1 + (boffset / 96)), &cfg->idx, "MRF_INDEX")) return "Bitmap read error"; +#if defined(be32toh) // Change to host endian for (int i = 0; i < 4; i++) line[i] = be32toh(line[i]); +#endif // The relocated block number for the original index record uint64_t blockn = block_count(line, static_cast(boffset % 96)); @@ -394,8 +397,11 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { if (sizeof(range_t) != vfile_pread(r, dst, offset, &cfg->idx, "MRF_INDEX")) return "Read error"; +#if defined(be64toh) idx->offset = be64toh(idx->offset); idx->size = be64toh(idx->size); +#endif + return nullptr; } From 759050be121c7e7ae84955b9a1db90cef4ed51c8 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 29 Oct 2019 17:03:51 -0700 Subject: [PATCH 65/85] Map the selector to file name prefix --- src/mod_mrf.cpp | 123 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 90 insertions(+), 33 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index f4c4d3e..e180613 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -11,6 +11,7 @@ #include #include #include +#include using namespace std; NS_AHTSE_USE @@ -29,15 +30,20 @@ NS_AHTSE_USE #define bsc __builtin_popcount #endif -// The next two functions are from the mrf/mrf_apps/can program +// The next few functions are from the mrf/mrf_apps/can program +// Check a specific bit position in a canned index header line +// The bit position is [0, 95] and the first 32bit value is skipped inline bool is_on(uint32_t *values, int bit) { return 0 != (values[1 + bit / 32] & (static_cast(1) << bit % 32)); } +// Canned index file header size +// 16 byte prefix + 96 bit records in 128 bit lines static inline uint64_t hsize(uint64_t in_size) { return 16 + 16 * ((96 * BSZ - 1 + in_size) / (96 * BSZ)); } +// Packed block count, for a bit position in a line inline uint32_t block_count(uint32_t *values, int bit) { if (!is_on(values, bit)) return NPOS; @@ -47,6 +53,12 @@ inline uint32_t block_count(uint32_t *values, int bit) { bsc(values[1 + (bit / 32)] & ((1ULL << (bit % 32)) - 1))); } +// How do we map M param mapping to a file name? +enum mappings{ + MAPM_NONE = 0, + MAPM_PREFIX // The M value prefixes the file name, both index and data +}; + struct mrf_conf { // array of guard regexp, one of them has to match apr_array_header_t *arr_rxp; @@ -70,6 +82,9 @@ struct mrf_conf { // If set, file handles are not held open int dynamic; + // How do we map M param mapping to a file name? + int mmapping; + // the canned index header size, or 0 for normal index uint64_t can_hsize; }; @@ -127,6 +142,17 @@ static const char *parse_sources(cmd_parms *cmd, const char *src, #define parse_redirects(cmd, src, arr) parse_sources(cmd, src, arr, true) +// +// This function sets the MRF specific parameters +// The raster size is defined using the normal libahtse parameters +// Unique directives: +// IndexFile : May be local paths or indirect, if prefixed by :// +// DataFile +// Redirect : Old style redirects, only if DataFile is not present +// RetryCount : // For indirect redirect range requests +// EmptyTile : +// Dynamic On : If the file handles are not to be hold +// static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) { ap_assert(sizeof(apr_off_t) == 8); @@ -200,6 +226,16 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) if ((line = apr_table_get(kvp, "CannedIndex")) && getBool(line)) c->can_hsize = hsize((c->raster.size.z + c->raster.rsets[0].tiles) * 16); + // What parameters we need for M mapping ? + if ((line = apr_table_get(kvp, "MMapping"))) { + if (apr_strnatcmp(line, "prefix")) { + // Direct mapping means M becomes the prefix for the file name, no folder + c->mmapping = MAPM_PREFIX; + } + else + return "Unknown value for MMapping"; + } + return NULL; } @@ -223,65 +259,68 @@ static const vfile_t *pick_source(const apr_array_header_t *sources, range_t *in // An open file handle and the matching file name, to be used as a note struct file_note { - const vfile_t *fh; + const char *fname; apr_file_t *pfh; }; // // Open or retrieve a connection cached file handle +// This is a real file // Do no close the returned file handle, it will be removed when the connection drops // // Only one opened handle exists per connection/token pair // This may lead to less caching, but avoids having too many opened files // -static apr_status_t openConnFile(request_rec *r, apr_file_t **ppfh, const vfile_t *fh, +static apr_status_t openConnFile(request_rec *r, apr_file_t **ppfh, const char *fname, const char *token, apr_int32_t extra_open_flags = 0) { apr_table_t *conn_notes = r->connection->notes; - file_note *fn = (file_note *)apr_table_get(conn_notes, token); - if (fn && fn->fh == fh) { // Match, return the handle - *ppfh = fn->pfh; + file_note *fnote = (file_note *)apr_table_get(conn_notes, token); + if (fnote && !apr_strnatcmp(fnote->fname, fname)) { // Match, return the handle + *ppfh = fnote->pfh; return APR_SUCCESS; } // Use the connection pool, it will close the file when it gets dropped apr_pool_t *pool = r->connection->pool; - if (!fn) { // new connection file - fn = reinterpret_cast(apr_pcalloc(pool, sizeof(file_note))); + if (!fnote) { // new connection file + fnote = reinterpret_cast(apr_pcalloc(pool, sizeof(file_note))); } else { // Not the right file, clean it up apr_table_unset(conn_notes, token); // Does not remove the storage - apr_file_close(fn->pfh); + apr_file_close(fnote->pfh); } - apr_status_t stat = apr_file_open(ppfh, fh->name, open_flags || extra_open_flags, 0, pool); + apr_status_t stat = apr_file_open(ppfh, fname, open_flags || extra_open_flags, 0, pool); if (APR_SUCCESS != stat) return stat; // Update the note and hook it up before returning - fn->fh = fh; - fn->pfh = *ppfh; - apr_table_setn(conn_notes, token, (const char *)fn); + fnote->fname = apr_pstrdup(pool, fname); + fnote->pfh = *ppfh; + apr_table_setn(conn_notes, token, (const char *)fnote); return APR_SUCCESS; } // Like pread, except not really thread safe +// Range reads are done if file starts with :// +// Range offset is offset, size is mgr->size // The token is used for connection caching + static int vfile_pread(request_rec *r, storage_manager &mgr, - apr_off_t offset, const vfile_t *fh, const char *token = "MRF_DATA") + apr_off_t offset, const char *fname, const char *token = "MRF_DATA") { auto cfg = get_conf(r, &mrf_module); - const char *name = fh->name; + const char *name = fname; bool redirect = (strlen(name) > 3 && name[0] == ':' && name[1] == '/'); - if (redirect) { // Remote file, just use a range request // TODO: S3 authorized requests // Skip the ":/" used to mark a redirect - name = fh->name + 2; + name = fname + 2; ap_filter_rec_t *receive_filter = ap_get_output_filter_handle("Receive"); if (!receive_filter) { @@ -337,7 +376,7 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, if (cfg->dynamic) stat = apr_file_open(&pfh, name, open_flags, 0, r->pool); else - stat = openConnFile(r, &pfh, fh, token, APR_FOPEN_BUFFERED); + stat = openConnFile(r, &pfh, fname, token, APR_FOPEN_BUFFERED); if (stat != APR_SUCCESS) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, @@ -360,9 +399,9 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, return (mgr.size = static_cast(sz)); } -// Indirect read of index, returns error message or null +// read index, returns error message or null // MRF index file is network order -static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { +static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset, const char *fname) { auto cfg = get_conf(r, &mrf_module); storage_manager dst(idx, sizeof(range_t)); @@ -374,7 +413,7 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { uint32_t line[4]; storage_manager lmgr(line, 16); if (16 != vfile_pread(r, lmgr, - 16 * (1 + (boffset / 96)), &cfg->idx, "MRF_INDEX")) + 16 * (1 + (boffset / 96)), fname, "MRF_INDEX")) return "Bitmap read error"; #if defined(be32toh) @@ -394,7 +433,7 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { offset = cfg->can_hsize + blockn * BSZ + offset % BSZ; } - if (sizeof(range_t) != vfile_pread(r, dst, offset, &cfg->idx, "MRF_INDEX")) + if (sizeof(range_t) != vfile_pread(r, dst, offset, fname, "MRF_INDEX")) return "Read error"; #if defined(be64toh) @@ -416,6 +455,27 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset) { return HTTP_INTERNAL_SERVER_ERROR; \ } +// Change the file name depending on the configuration +// Returns nullptr if something went wrong +const char *apply_mmapping(request_rec *r, const sz *tile, const char *fname) +{ + auto cfg = get_conf(r, &mrf_module); + if (cfg->mmapping == MAPM_NONE || tile->z == 0) + return fname; + + // Switch to C++ + string ret_fname(fname); + switch (cfg->mmapping) { + case MAPM_PREFIX: + size_t bnamepos = ret_fname.find_last_of("/"); + if (bnamepos == string::npos) + return NULL; + ret_fname.insert(bnamepos + 1, apr_ltoa(r->pool, static_cast(tile->z))); + break; + } + return apr_pstrdup(r->pool, ret_fname.c_str()); +} + static int handler(request_rec *r) { if (r->method_number != M_GET) return DECLINED; @@ -443,7 +503,7 @@ static int handler(request_rec *r) { // We can ignore the error on this one, defaults to zero // The parameter before the level can't start with a digit for an extra-dimensional MRF - if (raster.size.z != 1 && tokens->nelts) + if (tokens->nelts && (raster.size.z != 1 || cfg->mmapping != MAPM_NONE)) tile.z = apr_atoi64(*(char **)apr_array_pop(tokens)); // Don't allow access to levels less than zero, send the empty tile instead @@ -461,19 +521,16 @@ static int handler(request_rec *r) { + level->w * (tile.z * level->h + tile.y) + tile.x); range_t index; - const char *message; - SERR_IF((message = read_index(r, &index, tidx_offset)), - message); + const char *idx_fname = apply_mmapping(r, &tile, cfg->idx.name); + const char *message = read_index(r, &index, tidx_offset, idx_fname); + SERR_IF(message, message); // MRF index record is in network order if (index.size < 4) // Need at least four bytes for signature check return sendEmptyTile(r, raster.missing); - if (MAX_TILE_SIZE < index.size) { // Tile is too large, log and send error code - ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, "Tile too large in %s", - cfg->idx.name); - return HTTP_INTERNAL_SERVER_ERROR; - } + SERR_IF(MAX_TILE_SIZE < index.size, apr_pstrcat(r->pool, + "Tile too large found in ", idx_fname, NULL)); // Check for conditional ETag here, no need to get the data char ETag[16]; @@ -486,14 +543,14 @@ static int handler(request_rec *r) { // Now for the data part const vfile_t *src = pick_source(cfg->source, &index); - const char *name = (src && src->name) ? src->name : nullptr; + const char *name = (src && src->name) ? apply_mmapping(r, &tile, src->name) : nullptr; SERR_IF(!name, apr_psprintf(r->pool, "No data file configured for %s", r->uri)); apr_size_t size = static_cast(index.size); storage_manager img(apr_palloc(r->pool, size), size); SERR_IF(!img.buffer, "Memory allocation error in mod_mrf"); - SERR_IF(img.size != vfile_pread(r, img, index.offset, src), + SERR_IF(img.size != vfile_pread(r, img, index.offset, name), "Data read error"); // Looks fine, set the outgoing etag and then the image From b183b426926cb5e7b11615621a35e998ec84f221 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 29 Oct 2019 17:47:05 -0700 Subject: [PATCH 66/85] Minor bug fixes, initial test passed. --- src/mod_mrf.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index e180613..e8c47d9 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -228,7 +228,7 @@ static const char *file_set(cmd_parms *cmd, void *dconf, const char *arg) // What parameters we need for M mapping ? if ((line = apr_table_get(kvp, "MMapping"))) { - if (apr_strnatcmp(line, "prefix")) { + if (!apr_strnatcmp(line, "prefix")) { // Direct mapping means M becomes the prefix for the file name, no folder c->mmapping = MAPM_PREFIX; } @@ -516,9 +516,11 @@ static int handler(request_rec *r) { rset *level = raster.rsets + tile.l; REQ_ERR_IF(tile.x >= level->w || tile.y >= level->h); + // Force single z if that's how the MRF is set up, maybe file name mapping applies + apr_int64_t tz = (raster.size.z != 1) ? tile.z : 0; // Offset of the index entry for this tile apr_off_t tidx_offset = sizeof(range_t) * (level->tiles + - + level->w * (tile.z * level->h + tile.y) + tile.x); + + level->w * (tz * level->h + tile.y) + tile.x); range_t index; const char *idx_fname = apply_mmapping(r, &tile, cfg->idx.name); From 6b1087d6722ffcb44df45b8fb02919531d5316ed Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 29 Oct 2019 17:52:45 -0700 Subject: [PATCH 67/85] Document new option Should write a HOWTO to make it easier to understand --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index b7af77d..b665c38 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,13 @@ AHTSE Control Directives for this module are: when the MRF files are changed at run-time, avoiding stale or even broken content. MRF in-place modification do not require this flag because the old content is still available + ***MMapping prefix*** + - Optional, controls the mapping of the M parameter to data source. The only value currently + implemented is _prefix_, which means that the M, as a decimal number will be added right in front of + the basename of the file, both the Index and Data. The range based data file split still applies, + each part will be prefixed by the M value + + ***CannedIndex On*** - Optional, flags the index file as a canned format index, see mrf_apps/can.cpp. This is a dense format index that can be much smaller. Should be used only when needed, and not recommended when From 7a0e70c4bc558a5e82a64c823033c2c0b073f1ef Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 14 Nov 2019 11:35:37 -0800 Subject: [PATCH 68/85] Make only the unmodified data and index files subject to dynamic flag This will improve performance for m-dimensional tile services, while still preserving the fast response for the "current" data --- src/mod_mrf.cpp | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index e8c47d9..2eedcc9 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -1,5 +1,5 @@ /* -* An OnEarth module that serves tiles from an MRF +* An AHTSE module that serves tiles from an MRF * Lucian Plesea * (C) 2016-2019 */ @@ -311,7 +311,7 @@ static apr_status_t openConnFile(request_rec *r, apr_file_t **ppfh, const char * static int vfile_pread(request_rec *r, storage_manager &mgr, apr_off_t offset, const char *fname, const char *token = "MRF_DATA") { - auto cfg = get_conf(r, &mrf_module); + auto cfg = get_conf(r, &mrf_module); const char *name = fname; bool redirect = (strlen(name) > 3 && name[0] == ':' && name[1] == '/'); @@ -372,9 +372,22 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, apr_file_t *pfh; apr_status_t stat; - // Don't keep handles open - if (cfg->dynamic) - stat = apr_file_open(&pfh, name, open_flags, 0, r->pool); + int dynamic = 0; + // Keep handles open, except if dynamic is on and fname is the default + if (cfg->dynamic) { + if (!apr_strnatcmp(token, "MRF_DATA")) { + // Only single data file, unmodified can be dynamic + if (1 == cfg->source->nelts) { + dynamic = !apr_strnatcmp(fname, APR_ARRAY_IDX(cfg->source, 0, vfile_t).name); + } + } + else { // Only unmodified index name can be dynamic + dynamic = !apr_strnatcmp(fname, cfg->idx.name); + } + } + + if (dynamic) + stat = apr_file_open(&pfh, fname, open_flags, 0, r->pool); else stat = openConnFile(r, &pfh, fname, token, APR_FOPEN_BUFFERED); @@ -392,7 +405,7 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, sz = 0; } - if (cfg->dynamic) + if (dynamic) apr_file_close(pfh); // Don't close non-dynamic mode handles, they are reused From c815eb2c0f7de6f8ca12f89c7f50edfbcbbaa914 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 14 Nov 2019 12:00:48 -0800 Subject: [PATCH 69/85] Not a server configuration error for requesting invalid M values --- src/mod_mrf.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 2eedcc9..1a535cb 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -538,7 +538,12 @@ static int handler(request_rec *r) { range_t index; const char *idx_fname = apply_mmapping(r, &tile, cfg->idx.name); const char *message = read_index(r, &index, tidx_offset, idx_fname); - SERR_IF(message, message); + if (message) { + if (apr_strnatcmp(idx_fname, cfg->idx.name)) + SERR_IF(message, message) + else + REQ_ERR_IF(message); + } // MRF index record is in network order if (index.size < 4) // Need at least four bytes for signature check From fb92ce3a53a2e522c100109dcbe33301480cbfcd Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 14 Nov 2019 12:01:42 -0800 Subject: [PATCH 70/85] dangling else --- src/mod_mrf.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 1a535cb..876329c 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -539,10 +539,12 @@ static int handler(request_rec *r) { const char *idx_fname = apply_mmapping(r, &tile, cfg->idx.name); const char *message = read_index(r, &index, tidx_offset, idx_fname); if (message) { - if (apr_strnatcmp(idx_fname, cfg->idx.name)) - SERR_IF(message, message) - else + if (apr_strnatcmp(idx_fname, cfg->idx.name)) { + SERR_IF(message, message); + } + else { REQ_ERR_IF(message); + } } // MRF index record is in network order From 7f11ef29e4d35b6b068262b12da69b18eb657eaf Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 14 Nov 2019 13:15:11 -0800 Subject: [PATCH 71/85] parameters are accepted by default --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index b665c38..2b2539d 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,6 @@ Apache configuration directives: - Empty lines, lines that start with # are considered comments - Unknown directives are ignored - **MRF_AllowParams On** - - If set, the MRF module allows get requests with parameters. Default is Off - AHTSE Control Directives for this module are: ***DataFile path start_offset size*** From b4ab77ac0d022d209e02abfd87bfb9be4cab8ede Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 14 Nov 2019 13:24:44 -0800 Subject: [PATCH 72/85] fix condition for 500 or 400 errors --- src/mod_mrf.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 876329c..5481194 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -538,13 +538,11 @@ static int handler(request_rec *r) { range_t index; const char *idx_fname = apply_mmapping(r, &tile, cfg->idx.name); const char *message = read_index(r, &index, tidx_offset, idx_fname); - if (message) { - if (apr_strnatcmp(idx_fname, cfg->idx.name)) { + if (message) { // Fatal error + if (!apr_strnatcmp(idx_fname, cfg->idx.name)) { SERR_IF(message, message); } - else { - REQ_ERR_IF(message); - } + REQ_ERR_IF(message); } // MRF index record is in network order From 460636bd35e0c6a922498377a36b9899cd75f42b Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 11 Feb 2020 15:18:34 -0800 Subject: [PATCH 73/85] Set up vs2019, x64, Debug --- mod_mrf.vcxproj | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index 7de8a0f..b7d79ff 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -34,32 +34,32 @@ {C7E01836-333F-490D-A3B5-3554A8935040} Win32Proj mod_mrf - 10.0.17134.0 + 10.0 DynamicLibrary true - v141 + v142 NotSet DynamicLibrary true - v141 + v142 NotSet Application false - v141 + v142 true NotSet Application false - v141 + v142 true NotSet @@ -87,6 +87,8 @@ true .so + $(SolutionDir)$(Configuration)\ + $(Configuration)\ false @@ -108,7 +110,7 @@ Windows true \Apache24\lib - libahtse.lib;libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) + libahtse.lib;libhttpd.lib;libapr.lib;%(AdditionalDependencies) /EXPORT:mrf_module,@1 $(OutDir)$(TargetName)$(TargetExt) @@ -130,12 +132,12 @@ Windows true \Apache24\lib - libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) + libahtse.lib;libhttpd.lib;libapr.lib;%(AdditionalDependencies) /EXPORT:mrf_module,@1 $(OutDir)$(TargetName)$(TargetExt) - copy /y $(OutDir)$(TargetName)$(TargetExt) \Apache24\modules + copy /y $(TargetPath) \Apache24\modules From 7caa1b1e2e01cae9e56c1550201eadf231470fd2 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 8 Dec 2020 18:42:14 -0800 Subject: [PATCH 74/85] Matching libahtse lerc1 branch --- src/mod_mrf.cpp | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 5481194..e11acc0 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -102,13 +102,6 @@ static void *create_dir_config(apr_pool_t *p, char *dummy) { return c; } -static const char *set_regexp(cmd_parms *cmd, - mrf_conf *c, - const char *pattern) -{ - return add_regexp_to_array(cmd->pool, &c->arr_rxp, pattern); -} - // Parse a comma separated list of sources, add the entries to the array // Source may include offset and size, white space separated static const char *parse_sources(cmd_parms *cmd, const char *src, @@ -579,6 +572,14 @@ static int handler(request_rec *r) { } static const command_rec cmds[] = { + AP_INIT_TAKE1( + "MRF_RegExp", + (cmd_func)set_regexp, + 0, // Self-pass argument + ACCESS_CONF, // availability + "Regular expression that the URL has to match. At least one is required." + ), + AP_INIT_FLAG( "MRF_Indirect", (cmd_func) ap_set_flag_slot, @@ -587,14 +588,6 @@ static const command_rec cmds[] = { "If set, this configuration only responds to subrequests" ), - AP_INIT_TAKE1( - "MRF_RegExp", - (cmd_func) set_regexp, - 0, // Self-pass argument - ACCESS_CONF, // availability - "Regular expression that the URL has to match. At least one is required." - ), - AP_INIT_TAKE1( "MRF_ConfigurationFile", (cmd_func) file_set, // Callback From 17c4c6f181f25bd92c44df9f236987b8d003f2e5 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Fri, 11 Dec 2020 12:30:10 -0800 Subject: [PATCH 75/85] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b2539d..0285ecb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # mod_mrf [AHTSE](https://github.com/lucianpls/AHTSE) -An apache module that serves tiles directly from a local MRF, 2D or 3D. +An apache module that serves tiles directly from a local MRF, 2D or 3D. +With the MRF data on a local SSD, this module average request latency is .25ms (as measured by httpd), and reaches request rates above 20000 req/sec on a single core. + Apache configuration directives: **MRF_RegExp** From 9113ef4e6e6efb3abdf82c17b781641ec8a06f36 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Fri, 11 Dec 2020 12:30:53 -0800 Subject: [PATCH 76/85] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0285ecb..6a9c0ce 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # mod_mrf [AHTSE](https://github.com/lucianpls/AHTSE) -An apache module that serves tiles directly from a local MRF, 2D or 3D. -With the MRF data on a local SSD, this module average request latency is .25ms (as measured by httpd), and reaches request rates above 20000 req/sec on a single core. +An apache module that serves tiles directly from a local MRF, 2D or 3D. +With the MRF data on a local SSD, this module average tile request latency is .25ms (as measured by httpd), and reaches request rates above 20000 req/sec on a single core. Apache configuration directives: From 41ddb73248687f87f56f24e6be3de35ec631fd9a Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 21 Jun 2021 20:21:29 -0700 Subject: [PATCH 77/85] slightly better makefiles --- src/Makefile | 20 +++++++++++--------- src/Makefile.lcl.example | 9 +++++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Makefile b/src/Makefile index 3bb082f..662e9e5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,6 +1,6 @@ -TGT = mod_mrf -C_SRC = $(TGT).cpp -FILES = $(C_SRC) +MODULE = mod_mrf +TARGET = .libs/$(MODULE).so +C_SRC = $(MODULE).cpp DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE $(DEBUG) @@ -10,16 +10,18 @@ DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE $(DEBUG) # LIBTOOL, DEBUG ... include Makefile.lcl -TARGET = .libs/$(TGT).so # Can't use apxs to build c++ modules # The options used here might depend on how apache was built -$(TARGET) : $(FILES) - $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(INCLUDES) -pthread -c -o $(TGT).lo $(C_SRC) && touch $(TGT).slo - $(LIBTOOL) --silent --mode=link g++ -o $(TGT).la -rpath $(DEST) -module -avoid-version $(TGT).lo + +default : $(TARGET) + +$(TARGET) : $(C_SRC) + $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(EXTRA_INCLUDES) -pthread -c -o $(MODULE).lo $< && touch $(MODULE).slo + $(LIBTOOL) --silent --mode=link g++ -o $(MODULE).la -rpath $(DEST) -module -avoid-version $(MODULE).lo install : $(TARGET) - $(SUDO) cp $(TARGET) $(DEST) + $(SUDO) $(CP) $(TARGET) $(DEST) clean : - rm -rf .libs *.{o,lo,slo,la} + $(RM) -r .libs *.{o,lo,slo,la} diff --git a/src/Makefile.lcl.example b/src/Makefile.lcl.example index 7bbb3f4..350d129 100644 --- a/src/Makefile.lcl.example +++ b/src/Makefile.lcl.example @@ -1,3 +1,8 @@ -INCLUDES = -I /usr/include/httpd -I /usr/include/apr-1 -MOD_PATH = /etc/httpd/modules +APXS = apxs +includedir = $(shell $(APXS) -q includedir 2>/dev/null) +EXTRA_INCLUDES = $(shell $(APXS) -q EXTRA_INCLUDES 2>/dev/null) +LIBTOOL = $(shell $(APXS) -q LIBTOOL 2>/dev/null) +# SUDO = sudo +CP = cp +DEST = /etc/httpd/modules From 95cbedc9691d7db95719dae6abb5712f0a0bebb4 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 30 Jun 2021 12:27:59 -0700 Subject: [PATCH 78/85] Update makefiles --- src/Makefile | 3 ++- src/Makefile.lcl.example | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 662e9e5..031c1ea 100644 --- a/src/Makefile +++ b/src/Makefile @@ -9,7 +9,8 @@ DEFINES = -DLINUX -D_REENTRANT -D_GNU_SOURCE $(DEBUG) # INCLUDES, the gcc include commands for httpd, apr headers # LIBTOOL, DEBUG ... -include Makefile.lcl +MAKEOPT ?= Makefile.lcl +include $(MAKEOPT) # Can't use apxs to build c++ modules # The options used here might depend on how apache was built diff --git a/src/Makefile.lcl.example b/src/Makefile.lcl.example index 350d129..085e901 100644 --- a/src/Makefile.lcl.example +++ b/src/Makefile.lcl.example @@ -1,8 +1,9 @@ APXS = apxs +PREFIX ?= $(HOME) includedir = $(shell $(APXS) -q includedir 2>/dev/null) EXTRA_INCLUDES = $(shell $(APXS) -q EXTRA_INCLUDES 2>/dev/null) LIBTOOL = $(shell $(APXS) -q LIBTOOL 2>/dev/null) # SUDO = sudo CP = cp -DEST = /etc/httpd/modules +DEST = $(PREFIX)/modules From 420c89c844d15c2e06a448428a5a6d892b6460c2 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Wed, 30 Jun 2021 14:42:35 -0700 Subject: [PATCH 79/85] Add run path variable --- src/Makefile | 2 +- src/Makefile.lcl.example | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 031c1ea..eca2c6f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -19,7 +19,7 @@ default : $(TARGET) $(TARGET) : $(C_SRC) $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(EXTRA_INCLUDES) -pthread -c -o $(MODULE).lo $< && touch $(MODULE).slo - $(LIBTOOL) --silent --mode=link g++ -o $(MODULE).la -rpath $(DEST) -module -avoid-version $(MODULE).lo + $(LIBTOOL) --silent --mode=link g++ -o $(MODULE).la -rpath $(LIBEXECDIR) -module -avoid-version $(MODULE).lo install : $(TARGET) $(SUDO) $(CP) $(TARGET) $(DEST) diff --git a/src/Makefile.lcl.example b/src/Makefile.lcl.example index 085e901..fb75216 100644 --- a/src/Makefile.lcl.example +++ b/src/Makefile.lcl.example @@ -3,6 +3,7 @@ PREFIX ?= $(HOME) includedir = $(shell $(APXS) -q includedir 2>/dev/null) EXTRA_INCLUDES = $(shell $(APXS) -q EXTRA_INCLUDES 2>/dev/null) LIBTOOL = $(shell $(APXS) -q LIBTOOL 2>/dev/null) +LIBEXECDIR = \$(shell \$(APXS) -q libexecdir 2>/dev/null) # SUDO = sudo CP = cp From 1ab833fbf8b2b804aea69a1cd755587ea22db792 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 5 Jul 2021 13:18:10 -0700 Subject: [PATCH 80/85] separate libicd --- mod_mrf.vcxproj | 2 +- src/Makefile | 2 +- src/Makefile.lcl.example | 1 + src/mod_mrf.cpp | 16 +++++++++------- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index b7d79ff..007b2c6 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -132,7 +132,7 @@ Windows true \Apache24\lib - libahtse.lib;libhttpd.lib;libapr.lib;%(AdditionalDependencies) + libicd.lib;libahtse.lib;libhttpd.lib;libapr.lib;%(AdditionalDependencies) /EXPORT:mrf_module,@1 $(OutDir)$(TargetName)$(TargetExt) diff --git a/src/Makefile b/src/Makefile index eca2c6f..8e6e13a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -18,7 +18,7 @@ include $(MAKEOPT) default : $(TARGET) $(TARGET) : $(C_SRC) - $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(EXTRA_INCLUDES) -pthread -c -o $(MODULE).lo $< && touch $(MODULE).slo + $(LIBTOOL) --silent --mode=compile g++ -prefer-pic -O2 -Wall $(DEFINES) $(EXTRA_INCLUDES) -I $(EXP_INCLUDEDIR) -pthread -c -o $(MODULE).lo $< && touch $(MODULE).slo $(LIBTOOL) --silent --mode=link g++ -o $(MODULE).la -rpath $(LIBEXECDIR) -module -avoid-version $(MODULE).lo install : $(TARGET) diff --git a/src/Makefile.lcl.example b/src/Makefile.lcl.example index fb75216..0872f8b 100644 --- a/src/Makefile.lcl.example +++ b/src/Makefile.lcl.example @@ -4,6 +4,7 @@ includedir = $(shell $(APXS) -q includedir 2>/dev/null) EXTRA_INCLUDES = $(shell $(APXS) -q EXTRA_INCLUDES 2>/dev/null) LIBTOOL = $(shell $(APXS) -q LIBTOOL 2>/dev/null) LIBEXECDIR = \$(shell \$(APXS) -q libexecdir 2>/dev/null) +EXP_INCLUDEDIR = $(PREFIX)/include # SUDO = sudo CP = cp diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index e11acc0..ef638fe 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -15,6 +15,7 @@ using namespace std; NS_AHTSE_USE +NS_ICD_USE // Max count, should never happen #define NPOS 0xffffffff @@ -33,7 +34,7 @@ NS_AHTSE_USE // The next few functions are from the mrf/mrf_apps/can program // Check a specific bit position in a canned index header line // The bit position is [0, 95] and the first 32bit value is skipped -inline bool is_on(uint32_t *values, int bit) { +static inline bool is_on(uint32_t *values, int bit) { return 0 != (values[1 + bit / 32] & (static_cast(1) << bit % 32)); } @@ -44,7 +45,7 @@ static inline uint64_t hsize(uint64_t in_size) { } // Packed block count, for a bit position in a line -inline uint32_t block_count(uint32_t *values, int bit) { +static inline uint32_t block_count(uint32_t *values, int bit) { if (!is_on(values, bit)) return NPOS; return (values[0] + @@ -324,8 +325,8 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, // Get a buffer for the received image receive_ctx rctx; - rctx.buffer = mgr.buffer; - rctx.maxsize = mgr.size; + rctx.buffer = static_cast(mgr.buffer); + rctx.maxsize = static_cast(mgr.size); rctx.size = 0; // Data file is on a remote site a range request redirect with a range header @@ -402,7 +403,8 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, apr_file_close(pfh); // Don't close non-dynamic mode handles, they are reused - return (mgr.size = static_cast(sz)); + mgr.size = sz; + return static_cast(sz); } // read index, returns error message or null @@ -463,7 +465,7 @@ static const char *read_index(request_rec *r, range_t *idx, apr_off_t offset, co // Change the file name depending on the configuration // Returns nullptr if something went wrong -const char *apply_mmapping(request_rec *r, const sz *tile, const char *fname) +const char *apply_mmapping(request_rec *r, const sz5* tile, const char *fname) { auto cfg = get_conf(r, &mrf_module); if (cfg->mmapping == MAPM_NONE || tile->z == 0) @@ -497,7 +499,7 @@ static int handler(request_rec *r) { // Use a xyzc structure, with c being the level // Input order is M/Level/Row/Column, with M being optional - sz tile; + sz5 tile; memset(&tile, 0, sizeof(tile)); // Need at least three numerical arguments From 92278c7ba34f4697a81336dd17cbbc0dcb1c88c8 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Mon, 5 Jul 2021 13:25:50 -0700 Subject: [PATCH 81/85] fix gcc warnings --- src/mod_mrf.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index ef638fe..776add0 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -349,7 +349,7 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, if ((status != APR_SUCCESS || sr->status != HTTP_PARTIAL_CONTENT - || rctx.size != mgr.size) + || static_cast(rctx.size) != mgr.size) && (0 == tries--)) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, @@ -357,7 +357,7 @@ static int vfile_pread(request_rec *r, storage_manager &mgr, name, apr_time_now() - now); failed = true; } - } while (!failed && rctx.size != mgr.size); + } while (!failed && static_cast(rctx.size) != mgr.size); return rctx.size; } // Redirect read @@ -520,9 +520,9 @@ static int handler(request_rec *r) { tile.l += raster.skip; // Check for bad requests, outside of the defined bounds - REQ_ERR_IF(tile.l >= raster.n_levels); + REQ_ERR_IF(tile.l >= static_cast(raster.n_levels)); rset *level = raster.rsets + tile.l; - REQ_ERR_IF(tile.x >= level->w || tile.y >= level->h); + REQ_ERR_IF(tile.x >= static_cast(level->w) || tile.y >= static_cast(level->h)); // Force single z if that's how the MRF is set up, maybe file name mapping applies apr_int64_t tz = (raster.size.z != 1) ? tile.z : 0; @@ -565,7 +565,7 @@ static int handler(request_rec *r) { storage_manager img(apr_palloc(r->pool, size), size); SERR_IF(!img.buffer, "Memory allocation error in mod_mrf"); - SERR_IF(img.size != vfile_pread(r, img, index.offset, name), + SERR_IF(img.size != static_cast(vfile_pread(r, img, index.offset, name)), "Data read error"); // Looks fine, set the outgoing etag and then the image From ebe84e7d75da4f99bca32a12a7555060e3f4f72f Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Thu, 26 May 2022 17:39:23 -0700 Subject: [PATCH 82/85] VS 2022 project update --- mod_mrf.vcxproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index 007b2c6..f584796 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -40,26 +40,26 @@ DynamicLibrary true - v142 + v143 NotSet DynamicLibrary true - v142 + v143 NotSet Application false - v142 + v143 true NotSet Application false - v142 + v143 true NotSet From ce649e2f38b52339512876ccec6907833301be06 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 12 Dec 2023 12:02:03 -0800 Subject: [PATCH 83/85] Update mod_mrf.vcxproj --- mod_mrf.vcxproj | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index f584796..a39251a 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -89,6 +89,8 @@ .so $(SolutionDir)$(Configuration)\ $(Configuration)\ + \HTTPD\include;\HTTPD\include\apr-2;$(VC_IncludePath);$(WindowsSDK_IncludePath); + \HTTPD\lib;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) false @@ -125,19 +127,21 @@ Disabled WIN32;_DEBUG;_WINDOWS;%(PreprocessorDefinitions) true - \Apache24\include + + Default Windows true - \Apache24\lib - libicd.lib;libahtse.lib;libhttpd.lib;libapr.lib;%(AdditionalDependencies) + + + libicd.lib;libahtse.lib;libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) /EXPORT:mrf_module,@1 - $(OutDir)$(TargetName)$(TargetExt) - copy /y $(TargetPath) \Apache24\modules + copy /y $(TargetPath) \HTTPD\modules + Install to \HTTPD\modules From fc7d638898011fc2b24edef23e200cef43dc39c0 Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Sun, 30 Jun 2024 18:31:23 -0700 Subject: [PATCH 84/85] update to vs x64 build --- macros.props | 15 ++++++++ mod_mrf.sln | 10 +++--- mod_mrf.vcxproj | 95 ++++++++----------------------------------------- 3 files changed, 34 insertions(+), 86 deletions(-) create mode 100644 macros.props diff --git a/macros.props b/macros.props new file mode 100644 index 0000000..d576359 --- /dev/null +++ b/macros.props @@ -0,0 +1,15 @@ + + + + + C:\httpd + + + + + + $(prefix) + true + + + \ No newline at end of file diff --git a/mod_mrf.sln b/mod_mrf.sln index 5f7196f..f91d834 100644 --- a/mod_mrf.sln +++ b/mod_mrf.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.28307.136 +# Visual Studio Version 17 +VisualStudioVersion = 17.10.34928.147 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mod_mrf", "mod_mrf.vcxproj", "{C7E01836-333F-490D-A3B5-3554A8935040}" EndProject @@ -13,12 +13,10 @@ Global Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|Win32.ActiveCfg = Debug|Win32 - {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|Win32.Build.0 = Debug|Win32 + {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|Win32.ActiveCfg = Debug|x64 {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|x64.ActiveCfg = Debug|x64 {C7E01836-333F-490D-A3B5-3554A8935040}.Debug|x64.Build.0 = Debug|x64 - {C7E01836-333F-490D-A3B5-3554A8935040}.Release|Win32.ActiveCfg = Release|Win32 - {C7E01836-333F-490D-A3B5-3554A8935040}.Release|Win32.Build.0 = Release|Win32 + {C7E01836-333F-490D-A3B5-3554A8935040}.Release|Win32.ActiveCfg = Release|x64 {C7E01836-333F-490D-A3B5-3554A8935040}.Release|x64.ActiveCfg = Release|x64 {C7E01836-333F-490D-A3B5-3554A8935040}.Release|x64.Build.0 = Release|x64 EndGlobalSection diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index a39251a..346858d 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -1,18 +1,10 @@  - - Debug - Win32 - Debug x64 - - Release - Win32 - Release x64 @@ -37,25 +29,12 @@ 10.0 - - DynamicLibrary - true - v143 - NotSet - DynamicLibrary true v143 NotSet - - Application - false - v143 - true - NotSet - Application false @@ -66,60 +45,28 @@ - - - - - - + + - - true - .so - \Apache24\include;$(VC_SourcePath); - true .so $(SolutionDir)$(Configuration)\ $(Configuration)\ - \HTTPD\include;\HTTPD\include\apr-2;$(VC_IncludePath);$(WindowsSDK_IncludePath); - \HTTPD\lib;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) - - - false + $(prefix)\include;$(prefix)\include\apr-2;$(VC_IncludePath);$(WindowsSDK_IncludePath); + $(prefix)\lib;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) false + $(prefix)\include;$(prefix)\include\apr-2;$(VC_IncludePath);$(WindowsSDK_IncludePath); + $(prefix)\lib;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) - - - NotUsing - Level3 - Disabled - WIN32;_DEBUG;_WINDOWS;%(PreprocessorDefinitions) - true - \Apache24\include - Default - - - Windows - true - \Apache24\lib - libahtse.lib;libhttpd.lib;libapr.lib;%(AdditionalDependencies) - /EXPORT:mrf_module,@1 - $(OutDir)$(TargetName)$(TargetExt) - - - copy /y $(TargetPath) \Apache24\modules - - NotUsing @@ -136,35 +83,18 @@ true - libicd.lib;libahtse.lib;libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) + libahtse.lib;libhttpd.lib;libapr-2.lib;%(AdditionalDependencies) /EXPORT:mrf_module,@1 - copy /y $(TargetPath) \HTTPD\modules - Install to \HTTPD\modules + copy /y $(TargetPath) $(prefix)\modules + Install to $(prefix)\modules - - - Level3 - Use - MaxSpeed - true - true - WIN32;NDEBUG;_WINDOWS;%(PreprocessorDefinitions) - true - - - Windows - true - true - true - - Level3 - Use + NotUsing MaxSpeed true true @@ -176,7 +106,12 @@ true true true + libahtse.lib;libhttpd.lib;libapr-2.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + + copy /y $(TargetPath) $(prefix)\modules + Install to $(prefix)\modules + From 91084426adf4bb70b0552441f47e6819218af1cf Mon Sep 17 00:00:00 2001 From: Lucian Plesea Date: Tue, 2 Jul 2024 11:56:40 -0700 Subject: [PATCH 85/85] Add exports as needed --- mod_mrf.vcxproj | 1 + src/mod_mrf.cpp | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mod_mrf.vcxproj b/mod_mrf.vcxproj index 346858d..dcb4a0e 100644 --- a/mod_mrf.vcxproj +++ b/mod_mrf.vcxproj @@ -107,6 +107,7 @@ true true libahtse.lib;libhttpd.lib;libapr-2.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + /EXPORT:mrf_module,@1 copy /y $(TargetPath) $(prefix)\modules diff --git a/src/mod_mrf.cpp b/src/mod_mrf.cpp index 776add0..deb5afc 100644 --- a/src/mod_mrf.cpp +++ b/src/mod_mrf.cpp @@ -9,9 +9,10 @@ #include #include +#include + #include #include -#include using namespace std; NS_AHTSE_USE @@ -595,7 +596,7 @@ static const command_rec cmds[] = { (cmd_func) file_set, // Callback 0, // Self-pass argument ACCESS_CONF, // availability - "The configuration file for this module" + "The AHTSE configuration file for this module" ), { NULL }