Big refactor, fixes for tests

src/bootstrap/bootstrap.cc

@@ -60,7 +60,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) {
     boost::function<plugin_t> creator;
     try {
         creator = boost::dll::import_alias<plugin_t>(lib_path / "libunderpass.so", "create_plugin", boost::dll::load_mode::append_decorations);
-        log_debug("Loaded plugin hotosm!");
+        log_debug("Loaded plugin!");
     } catch (std::exception &e) {
         log_debug("Couldn't load plugin! %1%", e.what());
         exit(0);
@@ -77,7 +77,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) {
     for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) {
         std::cout << "Counting geometries ... " << std::endl;
         int total = queryraw->getWaysCount(*table_it);
-        std::cout << "Total ways:" << total << std::endl;
+        std::cout << "Total: " << total << std::endl;
         if (total > 0) {
             int count = 0;
             long lastid = 0;
@@ -126,22 +126,10 @@ processWays(WayTask &wayTask, const std::string &tableName, const underpassconfi
     if (wayTask.processed > 0) {
         // Proccesing ways
         for (auto way = ways->begin(); way != ways->end(); ++way) {
-            // If it's closed polygon
-            if (way->isClosed()) {
-                log_debug("Way Id: %1%", way->id);
-
-                // Bad geometry
-                if (way->containsKey("building") && (boost::geometry::num_points(way->linestring) - 1 < 4 ||
-                    plugin->unsquared(way->linestring))
-                ) {
-                    auto status = ValidateStatus(*way);
-                    status.timestamp = boost::posix_time::microsec_clock::universal_time();
-                    status.source = "building";
-                    boost::geometry::centroid(way->linestring, status.center);
-                    task->query += queryvalidate->applyChange(status, badgeom);
-                }
+            auto status = plugin->checkWay(*way, "building");
+            for (auto status_it = status->status.begin(); status_it != status->status.end(); ++status_it) {
+                task->query += queryvalidate->applyChange(*status, *status_it);
             }
-
             // Fill the way_refs table
             if (!config.norefs) {
                 for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) {

src/data/pq.cc

@@ -25,7 +25,8 @@
 #include "data/pq.hh"
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/split.hpp>
-#include <boost/locale.hpp>
+#include <locale>
+#include <codecvt>
 #include <iostream>
 #include <string>
 #include <vector>
@@ -152,11 +153,24 @@ Pq::query(const std::string &query)
 {
     std::scoped_lock write_lock{pqxx_mutex};
     pqxx::work worker(*sdb);
-    pqxx::result result = worker.exec(query);
+    auto result = worker.exec(query);
     worker.commit();
     return result;
 }
 
+std::string Latin1ToUTF8(const std::string& latin1str) {
+    std::string utf8str;
+    for (char c : latin1str) {
+        if (static_cast<unsigned char>(c) <= 0x7F) {
+            utf8str.push_back(c);
+        } else {
+            utf8str.push_back(0xC0 | static_cast<unsigned char>(c) >> 6);
+            utf8str.push_back(0x80 | (static_cast<unsigned char>(c) & 0x3F));
+        }
+    }
+    return utf8str;
+}
+
 std::string
 Pq::escapedString(std::string text)
 {
@@ -180,7 +194,8 @@ Pq::escapedString(std::string text)
         }
         i++;
     }
-    return sdb->esc(newstr);
+
+    return sdb->esc(Latin1ToUTF8(newstr));
 }
 
 } // namespace pq

src/osm/osmchange.cc

@@ -130,6 +130,19 @@ OsmChangeFile::readChanges(const std::string &file)
     return true;
 }
 
+void
+OsmChangeFile::buildGeometriesFromNodeCache() {
+    for (auto it = std::begin(changes); it != std::end(changes); ++it) {
+        osmchange::OsmChange *change = it->get();
+        for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) {
+            osmobjects::OsmWay *way = wit->get();
+            for (auto lit = std::begin(way->refs); lit != std::end(way->refs); ++lit) {
+                boost::geometry::append(way->linestring, nodecache[*lit]);
+            }
+        }
+    }
+}
+
 bool
 OsmChangeFile::readXML(std::istream &xml)
 {
@@ -699,10 +712,7 @@ OsmChangeFile::validateNodes(const multipolygon_t &poly, std::shared_ptr<Validat
             std::vector<std::string> node_tests = {"building", "natural", "place", "waterway"};
             for (auto test_it = std::begin(node_tests); test_it != std::end(node_tests); ++test_it) {
                 if (node->containsKey(*test_it)) {
-                    auto status = plugin->checkPOI(*node, *test_it);
-                    if (status->status.size() > 0) {
-                        status->center = node->point;
-                    }
+                    auto status = plugin->checkNode(*node, *test_it);
                     totals->push_back(status);
                 }
             }
@@ -725,38 +735,8 @@ OsmChangeFile::validateWays(const multipolygon_t &poly, std::shared_ptr<Validate
             if (!way->priority) {
                 continue;
             }
-
-            // Geometry checks
-            auto status = std::make_shared<ValidateStatus>(*way);
-            if (way->containsKey("building")) {
-                // See if the way is a closed polygon
-                if (way->refs.front() == way->refs.back()) {
-                    status->timestamp = boost::posix_time::microsec_clock::universal_time();
-                    status->uid = way->uid;
-                    // Bad geometry
-                    if (boost::geometry::num_points(way->linestring) - 1 < 4 ||
-                        plugin->unsquared(way->linestring)
-                    ) {
-                        status->status.insert(badgeom);
-                    }
-                    if (status->status.size() > 0) {
-                        status->source = "building";
-                        boost::geometry::centroid(way->linestring, status->center);
-                    }
-                }
-            }
+            auto status = plugin->checkWay(*way, "building");
             totals->push_back(status);
-
-            // Semantic checks
-            std::vector<std::string> way_tests = {"building"};
-            for (auto test_it = way_tests.begin(); test_it != way_tests.end(); ++test_it) {
-                if (way->containsKey(*test_it)) {
-                    auto status = plugin->checkWay(*way, *test_it);
-                    status->source = *test_it;
-                    // boost::geometry::centroid(way->linestring, status->center);
-                    totals->push_back(status);
-                }
-            }
         }
     }
     return totals;

src/osm/osmchange.hh

@@ -245,6 +245,8 @@ class OsmChangeFile
     /// Delete any data not in the boundary polygon
     void areaFilter(const multipolygon_t &poly);
 
+    void buildGeometriesFromNodeCache();
+
 #ifdef LIBXML
     /// Called by libxml++ for each element of the XML file
     void on_start_element(const Glib::ustring &name,

src/raw/queryraw.cc

@@ -368,13 +368,13 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr<std::vector<OsmWay>> ways, std::m
     }
 }
 
-std::map<std::string, std::string> parseTagsString(const std::string& input) {
+std::map<std::string, std::string> parseTagsString(std::string input) {
+    input.pop_back();
     std::map<std::string, std::string> result;
     std::stringstream ss(input);
     std::string token;
     while (std::getline(ss, token, ',')) {
-        // Find the position of the arrow
-        size_t arrowPos = token.find(":");
+        size_t arrowPos = token.find_last_of(":");
         if (arrowPos != std::string::npos) {
             std::string key = token.substr(1, arrowPos - 1);
             std::string value = token.substr(arrowPos + 2);

src/replicator/threads.cc

@@ -248,7 +248,7 @@ startMonitorChanges(std::shared_ptr<replication::RemoteURL> &remote,
     boost::function<plugin_t> creator;
     try {
         creator = boost::dll::import_alias<plugin_t>(lib_path / "libunderpass.so", "create_plugin", boost::dll::load_mode::append_decorations);
-        log_debug("Loaded plugin hotosm!");
+        log_debug("Loaded plugin!");
     } catch (std::exception &e) {
         log_debug("Couldn't load plugin! %1%", e.what());
         exit(0);