Fix for parse tags from DB response, change 'overlaping' to 'overlapp…

…ing', less verbose

python/dbapi/api/raw.py

@@ -116,7 +116,6 @@ def listFeaturesQuery(
             "AND status = '{0}'".format(status) if (status) else "",
             "ORDER BY " + orderBy + " DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}".format(page * RESULTS_PER_PAGE_LIST) if page else ""),
         )
-        print("status", status)
         return query
 
 class Raw:

setup/underpass.sql

@@ -42,7 +42,7 @@ ALTER TABLE ONLY public.changesets
 DROP TYPE IF EXISTS public.objtype;
 CREATE TYPE public.objtype AS ENUM ('node', 'way', 'relation');
 DROP TYPE IF EXISTS public.status;
-CREATE TYPE public.status AS ENUM ('notags', 'complete', 'incomplete', 'badvalue', 'correct', 'badgeom', 'orphan', 'overlaping', 'duplicate');
+CREATE TYPE public.status AS ENUM ('notags', 'complete', 'incomplete', 'badvalue', 'correct', 'badgeom', 'orphan', 'overlapping', 'duplicate');
 
 CREATE TABLE IF NOT EXISTS public.validation (
     osm_id int8,

src/bootstrap/bootstrap.cc

@@ -75,7 +75,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) {
     };
 
     for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) {
-        std::cout << "Counting geometries ... " << std::endl;
+        std::cout << std::endl << "Counting geometries ... " << std::endl;
         int total = queryraw->getWaysCount(*table_it);
         std::cout << "Total: " << total << std::endl;
         if (total > 0) {

src/raw/queryraw.cc

@@ -36,6 +36,8 @@
 #include <boost/format.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/lexical_cast.hpp>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/json_parser.hpp>
 #include <map>
 #include <string>
 #include "utils/log.hh"
@@ -369,21 +371,24 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr<std::vector<OsmWay>> ways, std::m
 }
 
 std::map<std::string, std::string> parseTagsString(std::string input) {
-    input.pop_back();
-    std::map<std::string, std::string> result;
-    std::stringstream ss(input);
-    std::string token;
-    while (std::getline(ss, token, ',')) {
-        size_t arrowPos = token.find_last_of(":");
-        if (arrowPos != std::string::npos) {
-            std::string key = token.substr(1, arrowPos - 1);
-            std::string value = token.substr(arrowPos + 2);
-            key.erase(std::remove( key.begin(), key.end(), '\"' ),key.end());
-            value.erase(std::remove( value.begin(), value.end(), '\"' ),value.end());
-            result[key] = value;
-        }
+    // std::cout << "[INPUT] " << input << std::endl;
+    std::map<std::string, std::string> tags;
+    boost::property_tree::ptree pt;
+    try {
+        std::istringstream jsonStream(input);
+        boost::property_tree::read_json(jsonStream, pt);
+    } catch (const boost::property_tree::json_parser::json_parser_error& e) {
+        std::cerr << "Error parsing JSON: " << e.what() << std::endl;
+        return tags;
+    }
+    for (const auto& pair : pt) {
+        tags[pair.first] = pair.second.get_value<std::string>();
     }
-    return result;
+    // for (const auto& pair : tags) {
+    //     std::cout << pair.first << "=" << pair.second << std::endl;
+    // }
+    // std::cout << std::endl;
+    return tags;
 }
 
 std::list<std::shared_ptr<OsmWay>>

src/replicator/threads.cc

@@ -535,8 +535,8 @@ threadOsmChange(OsmChangeTask osmChangeTask)
                 for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) {
                     task.query += queryvalidate->applyChange(*it->get(), *status_it);
                 }
-                if (!it->get()->hasStatus(overlaping)) {
-                    task.query += queryvalidate->updateValidation(it->get()->osm_id, overlaping, "building");
+                if (!it->get()->hasStatus(overlapping)) {
+                    task.query += queryvalidate->updateValidation(it->get()->osm_id, overlapping, "building");
                 }
                 if (!it->get()->hasStatus(duplicate)) {
                     task.query += queryvalidate->updateValidation(it->get()->osm_id, duplicate, "building");

src/testsuite/libunderpass.all/stats-test.cc

@@ -237,7 +237,6 @@ class TestStats {
                     if (this->verbose) {
                         std::cout << "changeset: " << changestats->changeset << std::endl;
                     }
-                    // TODO: get values to test from YAML validation file
                     testStat(changestats, validation, "highway");
                     testStat(changestats, validation, "building");
                     testStat(changestats, validation, "humanitarian_building");

src/testsuite/libunderpass.all/test-playground.cc

@@ -18,44 +18,11 @@
 //
 
 #include <iostream>
-#include <map>
-#include <sstream>
-
-std::map<std::string, std::string> parseTagsString(const std::string& input) {
-    std::map<std::string, std::string> result;
-    std::stringstream ss(input);
-    std::string token;
-
-    while (std::getline(ss, token, ',')) {
-        // std::cout << token << std::endl;
-        // Remove leading and trailing whitespaces
-        // token.erase(token.find_first_not_of(" "), token.find_last_not_of(" ") + 1);
-        // std::cout << token << std::endl;
-        // Find the position of the arrow
-        size_t arrowPos = token.find("=>");
-        if (arrowPos != std::string::npos) {
-            std::string key = token.substr(0, arrowPos);
-            std::string value = token.substr(arrowPos + 2);
-            result[key] = value;
-        }
-    }
-
-    return result;
-}
 
 int main() {
-    std::string input = "\"fee\"=>\"no\", \"access\"=>\"yes\"";
-    std::map<std::string, std::string> resultMap = parseTagsString(input);
-
-    // Iterate over the map and print the key-value pairs
-    for (const auto& pair : resultMap) {
-        std::cout << pair.first << ": " << pair.second << std::endl;
-    }
-
     return 0;
 }
 
-
 // local Variables:
 // mode: C++
 // indent-tabs-mode: nil

src/testsuite/libunderpass.all/val-test.cc

@@ -91,6 +91,7 @@ osmobjects::OsmWay readOsmWayFromFile(std::string filename) {
     } else {
         log_debug("Couldn't load ! %1%", filespec);
     };
+    osmchange.buildGeometriesFromNodeCache();
     return *osmchange.changes.front().get()->ways.front().get();
 }
 
@@ -195,7 +196,7 @@ test_semantic(std::shared_ptr<Validate> &plugin) {
 
     // But it's complete
     status = plugin->checkNode(node, "building");
-    if (status->hasStatus(complete)) {
+    if (!status->hasStatus(incomplete)) {
         runtest.pass("Validate::checkNode(complete) [semantic building]");
     } else {
         runtest.fail("Validate::checkNode(complete) [semantic building]");
@@ -243,7 +244,7 @@ test_semantic(std::shared_ptr<Validate> &plugin) {
     // But it's complete
     way.addTag("building:material", "sponge");
     status = plugin->checkWay(way, "building");
-    if (status->hasStatus(complete)) {
+    if (!status->hasStatus(incomplete)) {
         runtest.pass("Validate::checkWay(bad value) [semantic building]");
     } else {
         runtest.fail("Validate::checkWay(bad value) [semantic building]");
@@ -418,29 +419,30 @@ test_geospatial(std::shared_ptr<Validate> &plugin)
     // }
 
     // Overlapping, duplicate
-    osmchange::OsmChangeFile osmfoverlaping;
+    osmchange::OsmChangeFile osmfoverlapping;
     const multipolygon_t poly;
     filespec = DATADIR;
     filespec += "/testdata/validation/rect-overlap-and-duplicate-building.osc";
     if (boost::filesystem::exists(filespec)) {
-        osmfoverlaping.readChanges(filespec);
+        osmfoverlapping.readChanges(filespec);
+        osmfoverlapping.buildGeometriesFromNodeCache();
     } else {
         log_debug("Couldn't load ! %1%", filespec);
     }
-    for (auto it = std::begin(osmfoverlaping.changes); it != std::end(osmfoverlaping.changes); ++it) {
+    for (auto it = std::begin(osmfoverlapping.changes); it != std::end(osmfoverlapping.changes); ++it) {
         osmchange::OsmChange *change = it->get();
         for (auto nit = std::begin(change->ways); nit != std::end(change->ways); ++nit) {
             osmobjects::OsmWay *way = nit->get();
             way->priority = true;
         }
     }
-    auto wayval = osmfoverlaping.validateWays(poly, plugin);
+    auto wayval = osmfoverlapping.validateWays(poly, plugin);
     for (auto sit = wayval->begin(); sit != wayval->end(); ++sit) {
         auto status = *sit->get();
-        if (status.hasStatus(overlaping)) {
-            runtest.pass("Validate::validateWays(overlaping) [geometry building]");
+        if (status.hasStatus(overlapping)) {
+            runtest.pass("Validate::validateWays(overlapping) [geometry building]");
         } else {
-            runtest.fail("Validate::validateWays(overlaping) [geometry building]");
+            runtest.fail("Validate::validateWays(overlapping) [geometry building]");
         }
         if (status.hasStatus(duplicate)) {
             runtest.pass("Validate::validateWays(duplicate) [geometry building]");
@@ -450,28 +452,28 @@ test_geospatial(std::shared_ptr<Validate> &plugin)
     }
 
     // No overlapping, no duplicate
-    osmchange::OsmChangeFile osmfnooverlaping;
+    osmchange::OsmChangeFile osmfnooverlapping;
     filespec = DATADIR;
     filespec += "/testsuite/testdata/validation/rect-no-overlap-and-duplicate-building.osc";
     if (boost::filesystem::exists(filespec)) {
-        osmfnooverlaping.readChanges(filespec);
+        osmfnooverlapping.readChanges(filespec);
     } else {
         log_debug("Couldn't load ! %1%", filespec);
     }
-    for (auto it = std::begin(osmfnooverlaping.changes); it != std::end(osmfnooverlaping.changes); ++it) {
+    for (auto it = std::begin(osmfnooverlapping.changes); it != std::end(osmfnooverlapping.changes); ++it) {
         osmchange::OsmChange *change = it->get();
         for (auto nit = std::begin(change->ways); nit != std::end(change->ways); ++nit) {
             osmobjects::OsmWay *way = nit->get();
             way->priority = true;
         }
     }
-    wayval = osmfnooverlaping.validateWays(poly, plugin);
+    wayval = osmfnooverlapping.validateWays(poly, plugin);
     for (auto sit = wayval->begin(); sit != wayval->end(); ++sit) {
         auto status = *sit->get();
-        if (!status.hasStatus(overlaping)) {
-            runtest.pass("Validate::validateWays(no overlaping) [geometry building]");
+        if (!status.hasStatus(overlapping)) {
+            runtest.pass("Validate::validateWays(no overlapping) [geometry building]");
         } else {
-            runtest.fail("Validate::validateWays(no overlaping) [geometry building]");
+            runtest.fail("Validate::validateWays(no overlapping) [geometry building]");
         }
         if (!status.hasStatus(duplicate)) {
             runtest.pass("Validate::validateWays(no duplicate) [geometry building]");

src/testsuite/testdata/validation/config/building.yaml

@@ -1,16 +1,12 @@
 
 config:
-  - minangle:
-    - 70
-  - maxangle:
-    - 110
-  - anglethreshold:
-    - 19
-  - overlaps:
+  - badgeom:
     - yes
-  - duplicate:
-    - yes
-  - enable_tag_values:
+  - badgeom_minangle:
+    - 89
+  - badgeom_maxangle:
+    - 91
+  - badvalue:
     - yes
 
 tags:

src/validate/defaultvalidation.cc

@@ -70,7 +70,9 @@ DefaultValidation::checkWay(const osmobjects::OsmWay &way, const std::string &ty
     yaml::Yaml tests = yamls[type];
     semantic::Semantic::checkWay(way, type, tests, status);
     geospatial::Geospatial::checkWay(way, type, tests, status);
-    boost::geometry::centroid(way.linestring, status->center);
+    if (way.linestring.size() > 2) {
+        boost::geometry::centroid(way.linestring, status->center);
+    }
     status->source = type;
     return status;
 }

src/validate/geospatial.cc

@@ -51,37 +51,44 @@ Geospatial::Geospatial() {}
 std::shared_ptr<ValidateStatus>
 Geospatial::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml::Yaml &tests, std::shared_ptr<ValidateStatus> &status)
 {
-    // On non-english numeric locales using decimal separator different than '.'
-    // this is necessary to parse double strings with std::stod correctly
-    // without loosing precision
-    // std::setlocale(LC_NUMERIC, "C");
-    setlocale(LC_NUMERIC, "C");
-
     if (way.action == osmobjects::remove) {
         return status;
     }
 
-    // These values are in the config section of the YAML file
     auto config = tests.get("config");
+    bool check_badgeom = config.get_value("badgeom") == "yes";    
+    // bool check_overlapping = config.get_value("overlapping") == "yes";
+    // bool check_duplicate = config.get_value("duplicate") == "yes";
+
+    if (check_badgeom) {
+        auto badgeom_minangle = config.get_value("badgeom_minangle");
+        auto badgeom_maxangle = config.get_value("badgeom_maxangle");
+        if (badgeom_minangle != "" && badgeom_maxangle != "") {
+            if (unsquared(way.linestring, std::stod(badgeom_minangle), std::stod(badgeom_maxangle))) {
+                status->status.insert(badgeom);
+            }
+        } else {
+            if (unsquared(way.linestring)) {
+                status->status.insert(badgeom);
+            }
+        }
 
-    // if (config.contains_value("overlaps", "yes")) {
+    }
+
+    // if (check_overlapping) {
         // auto allWays = context.getOverlappingWays();
         // if (overlaps(allWays, way)) {
-        //     status->status.insert(overlaping);
+        //     status->status.insert(overlapping);
         // }
     // }
 
-    // if (config.contains_value("duplicate", "yes")) {
+    // if (check_duplicate) {
         // auto allWays = context.getOverlappingWays();
         // if (overlaps(allWays, way)) {
-        //     status->status.insert(overlaping);
+        //     status->status.insert(overlapping);
         // }
     // }
 
-    if (unsquared(way.linestring)) {
-        status->status.insert(badgeom);
-    }
-
     return status;
 }
 
@@ -107,8 +114,6 @@ Geospatial::overlaps(const std::list<std::shared_ptr<osmobjects::OsmWay>> &allwa
 
 bool 
 Geospatial::duplicate(const std::list<std::shared_ptr<osmobjects::OsmWay>> &allways, osmobjects::OsmWay &way) {
-    // This test only applies to buildings, as highways often overlap.
-    // TODO: move logic to a config file
 #ifdef TIMING_DEBUG_X
     boost::timer::auto_cpu_timer timer("validate::duplicate: took %w seconds\n");
 #endif

src/validate/queryvalidate.cc

@@ -69,7 +69,7 @@ std::map<valerror_t, std::string> status_list = {
     {badvalue, "badvalue"},
     {correct, "correct"},
     {orphan, "orphan"},
-    {overlaping, "overlaping"},
+    {overlapping, "overlapping"},
     {duplicate, "duplicate"},
     {badgeom, "badgeom"}
 };