From b469f647fbbc30848cff7d2a51796032fe319afc Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Tue, 12 Mar 2024 16:38:12 -0300 Subject: [PATCH 1/9] Fix for areaFilter for relations, add indexes, remove unused code --- setup/db/indexes.sql | 3 + src/osm/osmchange.cc | 120 ++++++++++++--- src/osm/osmchange.hh | 2 + src/raw/queryraw.cc | 65 +------- .../libunderpass.all/areafilter-test.cc | 112 ++++++++------ src/testsuite/testdata/areafilter-test.osm | 142 ++++++------------ src/utils/geoutil.cc | 9 +- 7 files changed, 224 insertions(+), 229 deletions(-) diff --git a/setup/db/indexes.sql b/setup/db/indexes.sql index 8ab4f4287..d4646450c 100644 --- a/setup/db/indexes.sql +++ b/setup/db/indexes.sql @@ -1,8 +1,11 @@ CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC); CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC); +CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC); CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); +CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id); +CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id); CREATE INDEX nodes_version_idx ON public.nodes (version); CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index f6b7df24d..5c29ac61f 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -140,6 +140,80 @@ OsmChangeFile::buildGeometriesFromNodeCache() { if (way->isClosed()) { way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; } + waycache.insert(std::pair(way->id, std::make_shared(*way))); + } + for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + osmobjects::OsmRelation *relation = rit->get(); + buildRelationGeometry(*relation); + } + } +} + +void +OsmChangeFile::buildRelationGeometry(osmobjects::OsmRelation &relation) { + bool first = true; + bool isMultiPolygon = relation.isMultiPolygon(); + bool isMultiLineString = relation.isMultiLineString(); + if (isMultiPolygon || isMultiLineString) { + std::string innerGeoStr; + std::string geometry_str; + if (isMultiPolygon) { + geometry_str = "MULTIPOLYGON(("; + } else { + geometry_str = "MULTILINESTRING(("; + } + bool noWay = false; + for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { + if (!waycache.count(mit->ref)) { + noWay = true; + break; + } + auto way = waycache.at(mit->ref); + + if ((!isMultiPolygon && boost::geometry::num_points(way->linestring) == 0) || + (isMultiPolygon && boost::geometry::num_points(way->polygon) == 0) + ) { + noWay = true; + break; + } + std::stringstream ss; + std::string geometry; + + if (isMultiPolygon) { + ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); + geometry = ss.str(); + geometry.erase(0,8); + } else { + ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); + geometry = ss.str(); + geometry.erase(0,11); + } + + geometry.erase(geometry.size() - 1); + if (first && (mit->role == "outer")) { + geometry_str += geometry + ","; + } else { + if (mit->role == "inner") { + innerGeoStr += geometry + ","; + } else { + geometry_str += geometry + ","; + geometry_str += innerGeoStr; + innerGeoStr = ""; + } + } + } + if (innerGeoStr.size() > 0) { + geometry_str += innerGeoStr; + } + geometry_str.erase(geometry_str.size() - 1); + geometry_str += "))"; + + if (!noWay) { + if (isMultiPolygon) { + boost::geometry::read_wkt(geometry_str, relation.multipolygon); + } else { + boost::geometry::read_wkt(geometry_str, relation.multilinestring); + } } } } @@ -326,7 +400,6 @@ OsmChangeFile::on_start_element(const Glib::ustring &name, a.name); } } - // FIXME: is role mandatory? if (ref != -1 && type != osmobjects::osmtype_t::empty) { changes.back()->addMember(ref, type, role); } else { @@ -469,7 +542,7 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) if (poly.empty() || boost::geometry::within(node->point, poly)) { node->priority = true; nodecache[node->id] = node->point; - } if (!boost::geometry::within(node->point, poly)) { + } else if (!boost::geometry::within(node->point, poly)) { node->priority = false; } } @@ -484,27 +557,38 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { if (nodecache.count(*rit) && boost::geometry::within(nodecache[*rit], poly)) { way->priority = true; - continue; + if (boost::geometry::within(nodecache[*rit], poly)) { + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(nodecache[*rit]); + } + break; } } } + if (waycache.count(way->id)) { + waycache.at(way->id)->priority = way->priority; + } + } // Filter relations - // for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { - // OsmRelation *relation = rit->get(); - // if (poly.empty()) { - // relation->priority = true; - // } else { - // relation->priority = false; - // for (auto mit = std::begin(relation->members); mit != std::end(relation->members); ++mit) { - // if (nodecache.count(mit->ref) && boost::geometry::within(nodecache[mit->ref], poly)) { - // relation->priority = true; - // continue; - // } - // } - // } - // } + for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + OsmRelation *relation = rit->get(); + if (poly.empty()) { + relation->priority = true; + } else { + relation->priority = false; + for (auto mit = std::begin(relation->members); mit != std::end(relation->members); ++mit) { + if (waycache.count(mit->ref)) { + auto way = waycache.at(mit->ref); + if (way->priority) { + relation->priority = true; + break; + } + } + } + } + } } } diff --git a/src/osm/osmchange.hh b/src/osm/osmchange.hh index 18f186df0..5f6b9db63 100644 --- a/src/osm/osmchange.hh +++ b/src/osm/osmchange.hh @@ -246,6 +246,8 @@ class OsmChangeFile void areaFilter(const multipolygon_t &poly); void buildGeometriesFromNodeCache(); + void buildRelationGeometry(osmobjects::OsmRelation &relation); + #ifdef LIBXML /// Called by libxml++ for each element of the XML file diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index bb1db731f..ee8740f2f 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -614,70 +614,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { OsmRelation *relation = rel_it->get(); if (relation->priority) { - bool first = true; - bool isMultiPolygon = relation->isMultiPolygon(); - bool isMultiLineString = relation->isMultiLineString(); - if (isMultiPolygon || isMultiLineString) { - std::string innerGeoStr; - std::string geometry_str; - if (isMultiPolygon) { - geometry_str = "MULTIPOLYGON(("; - } else { - geometry_str = "MULTILINESTRING(("; - } - bool noWay = false; - for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { - if (!osmchanges->waycache.count(mit->ref)) { - noWay = true; - break; - } - auto way = osmchanges->waycache.at(mit->ref); - - if ((!isMultiPolygon && boost::geometry::num_points(way->linestring) == 0) || - (isMultiPolygon && boost::geometry::num_points(way->polygon) == 0) - ) { - noWay = true; - break; - } - std::stringstream ss; - std::string geometry; - - if (isMultiPolygon) { - ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); - geometry = ss.str(); - geometry.erase(0,8); - } else { - ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); - geometry = ss.str(); - geometry.erase(0,11); - } - - geometry.erase(geometry.size() - 1); - if (first && (mit->role == "outer")) { - geometry_str += geometry + ","; - } else { - if (mit->role == "inner") { - innerGeoStr += geometry + ","; - } else { - geometry_str += geometry + ","; - geometry_str += innerGeoStr; - innerGeoStr = ""; - } - } - } - if (innerGeoStr.size() > 0) { - geometry_str += innerGeoStr; - } - geometry_str.erase(geometry_str.size() - 1); - geometry_str += "))"; - if (!noWay) { - if (isMultiPolygon) { - boost::geometry::read_wkt(geometry_str, relation->multipolygon); - } else { - boost::geometry::read_wkt(geometry_str, relation->multilinestring); - } - } - } + osmchanges->buildRelationGeometry(*relation); } } } diff --git a/src/testsuite/libunderpass.all/areafilter-test.cc b/src/testsuite/libunderpass.all/areafilter-test.cc index 3a78f0140..d64ef63e7 100644 --- a/src/testsuite/libunderpass.all/areafilter-test.cc +++ b/src/testsuite/libunderpass.all/areafilter-test.cc @@ -33,13 +33,34 @@ class TestOsmChange : public osmchange::OsmChangeFile {}; int countFeatures(TestOsmChange &osmchange) { - int count = 0; + int nodeCount = 0; + int wayCount = 0; + int relCount = 0; for (auto cit = std::begin(osmchange.changes); cit != std::end(osmchange.changes); ++cit) { osmchange::OsmChange *testOsmChange = cit->get(); - count += testOsmChange->nodes.size(); - count += testOsmChange->ways.size(); + for (auto nit = std::begin(testOsmChange->nodes); nit != std::end(testOsmChange->nodes); ++nit) { + osmobjects::OsmNode *node = nit->get(); + if (node->priority) { + nodeCount++; + } + } + for (auto wit = std::begin(testOsmChange->ways); wit != std::end(testOsmChange->ways); ++wit) { + osmobjects::OsmWay *way = wit->get(); + if (way->priority) { + wayCount++; + } + } + for (auto rit = std::begin(testOsmChange->relations); rit != std::end(testOsmChange->relations); ++rit) { + osmobjects::OsmRelation *relation = rit->get(); + if (relation->priority) { + relCount++; + } + } } - return count; + // std::cout << "nodeCount: " << nodeCount << std::endl; + // std::cout << "wayCount: " << wayCount << std::endl; + // std::cout << "relCount: " << relCount << std::endl; + return nodeCount + wayCount + relCount; } int @@ -68,6 +89,11 @@ getPriority(TestOsmChange &osmchange, bool debug = false) { return result; } +void +clearChanges(TestOsmChange &osmchange) { + osmchange.changes.clear(); +} + int main(int argc, char *argv[]) { @@ -77,8 +103,12 @@ main(int argc, char *argv[]) boost::geometry::read_wkt("MULTIPOLYGON (((20 35, 45 20, 30 5, 10 10, 10 30, 20 35)))", polySmallArea); multipolygon_t polyHalf; boost::geometry::read_wkt("MULTIPOLYGON(((91.08473230447439 25.195528629552243,91.08475247411987 25.192143075605387,91.08932089882008 25.192152201214213,91.08927047470638 25.195501253482632,91.08473230447439 25.195528629552243)))", polyHalf); + multipolygon_t polyHalfSmall; + boost::geometry::read_wkt("MULTIPOLYGON(((91.08695983886719 25.195485830324174,91.08697056770325 25.192155906163805,91.08929872512817 25.192126781061106,91.08922362327574 25.195505246524604,91.08695983886719 25.195485830324174)))", polyHalfSmall); multipolygon_t polyEmpty; + // -- ChangeSets + // Small changeset in Bangladesh std::string changesetFile(DATADIR); changesetFile += "/testsuite/testdata/areafilter-test.osc"; @@ -131,52 +161,50 @@ main(int argc, char *argv[]) return 1; } - // OsmChange - Small area in North Africa - // FIXME - // osmchange.readChanges(osmchangeFile); - // osmchange.areaFilter(polySmallArea); - // if (countFeatures(osmchange) == 0) { - // runtest.pass("OsmChange areaFilter - false (small area)"); - // } else { - // runtest.fail("OsmChange areaFilter - false (small area)"); - // return 1; - // } + // -- OsmChanges - // OsmChange - Whole world + // OsmChange - Empty poly osmchange.readChanges(osmchangeFile); + osmchange.areaFilter(polyEmpty); + if (getPriority(osmchange) && countFeatures(osmchange) == 62) { + runtest.pass("OsmChange areaFilter - true (Empty poly)"); + } else { + runtest.fail("OsmChange areaFilter - true (Empty poly)"); + return 1; + } + + // OsmChange - Whole world + osmchange.buildGeometriesFromNodeCache(); osmchange.areaFilter(polyWholeWorld); - if (getPriority(osmchange) && countFeatures(osmchange) == 48) { + if (getPriority(osmchange) && countFeatures(osmchange) == 62) { runtest.pass("OsmChange areaFilter - true (whole world)"); } else { runtest.fail("OsmChange areaFilter - true (whole world)"); return 1; } - // Delete all changes - osmchange.areaFilter(polySmallArea); - - // OsmChange - Empty polygon - // FIXME - // osmchange.readChanges(osmchangeFile); - // osmchange.areaFilter(polyEmpty); - // if (getPriority(osmchange) && countFeatures(osmchange) == 48) { - // runtest.pass("OsmChange areaFilter - true (empty)"); - // } else { - // runtest.fail("OsmChange areaFilter - true (empty)"); - // return 1; - // } - // Delete all changes - // osmchange.areaFilter(polySmallArea); - - // OsmChange - Half area - // FIXME! - // osmchange.readChanges(osmchangeFile); - // osmchange.areaFilter(polyHalf); - // if (getPriority(osmchange, true) && countFeatures(osmchange) == 28) { - // runtest.pass("OsmChange areaFilter - true (half area)"); - // } else { - // runtest.fail("OsmChange areaFilter - true (half area)"); - // return 1; - // } + + // OsmChange - Small area + clearChanges(osmchange); + osmchange.readChanges(osmchangeFile); + osmchange.areaFilter(polyHalf); + if (countFeatures(osmchange) == 35) { + runtest.pass("OsmChange areaFilter - true (small area)"); + } else { + runtest.fail("OsmChange areaFilter - true (small area)"); + return 1; + } + + // OsmChange - Smaller area + clearChanges(osmchange); + osmchange.readChanges(osmchangeFile); + osmchange.areaFilter(polyHalfSmall); + if (countFeatures(osmchange) == 17) { + runtest.pass("OsmChange areaFilter - true (smaller area)"); + } else { + runtest.fail("OsmChange areaFilter - true (smaller area)"); + return 1; + } + } diff --git a/src/testsuite/testdata/areafilter-test.osm b/src/testsuite/testdata/areafilter-test.osm index 9c9ee6169..f0873cbd2 100644 --- a/src/testsuite/testdata/areafilter-test.osm +++ b/src/testsuite/testdata/areafilter-test.osm @@ -1,126 +1,46 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -129,8 +49,6 @@ - - @@ -139,8 +57,6 @@ - - @@ -149,8 +65,6 @@ - - @@ -159,8 +73,6 @@ - - @@ -169,8 +81,6 @@ - - @@ -187,8 +97,6 @@ - - @@ -197,8 +105,6 @@ - - @@ -207,5 +113,47 @@ - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/utils/geoutil.cc b/src/utils/geoutil.cc index a6b3d526c..6558672db 100644 --- a/src/utils/geoutil.cc +++ b/src/utils/geoutil.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -32,19 +32,12 @@ #include #include #include -// #include #include #include #include #include -// #include "boost/date_time/posix_time/posix_time.hpp" -// #include -// using namespace boost::posix_time; -// using namespace boost::gregorian; - #include "utils/geoutil.hh" -// #include "osm/changeset.hh" #include #include From 48be220a306dc43e2358720aa7176552cbf6c027 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 18 Mar 2024 18:52:34 -0300 Subject: [PATCH 2/9] Fixes for relation geometries + new tests --- src/bootstrap/bootstrap.cc | 4 +- src/osm/osmchange.cc | 184 ++++++++++---- src/osm/osmobjects.cc | 8 +- src/osm/osmobjects.hh | 5 +- src/raw/queryraw.cc | 69 ++++-- src/replicator/threads.cc | 3 +- .../libunderpass.all/areafilter-test.cc | 3 - src/testsuite/libunderpass.all/raw-test.cc | 49 ++-- src/testsuite/testdata/raw/raw-case-6.osc | 1 + src/testsuite/testdata/raw/raw-case-7.osc | 229 ++++++++++++++++++ src/underpass.cc | 4 +- 11 files changed, 457 insertions(+), 102 deletions(-) create mode 100644 src/testsuite/testdata/raw/raw-case-7.osc diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 1d84361fd..7358ed5fc 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -341,7 +341,7 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask) BootstrapTask task; int processed = 0; - auto relationval = std::make_shared>>(); + // auto relationval = std::make_shared>>(); // Proccesing relations for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) { @@ -355,7 +355,7 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask) ++processed; } } - // queryvalidate->relations(relval, task.query); + // queryvalidate->relations(relationval, task.query); task.processed = processed; const std::lock_guard lock(tasks_change_mutex); (*tasks)[taskIndex] = task; diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index 5c29ac61f..5c46ca96c 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -157,64 +157,149 @@ OsmChangeFile::buildRelationGeometry(osmobjects::OsmRelation &relation) { if (isMultiPolygon || isMultiLineString) { std::string innerGeoStr; std::string geometry_str; - if (isMultiPolygon) { - geometry_str = "MULTIPOLYGON(("; - } else { - geometry_str = "MULTILINESTRING(("; - } + bool noWay = false; + std::string linestring_tmp; + bool using_linestring_tmp = false; + linestring_t lastLinestring; + for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { - if (!waycache.count(mit->ref)) { - noWay = true; - break; - } - auto way = waycache.at(mit->ref); + if (mit->type == osmobjects::way) { - if ((!isMultiPolygon && boost::geometry::num_points(way->linestring) == 0) || - (isMultiPolygon && boost::geometry::num_points(way->polygon) == 0) - ) { - noWay = true; - break; - } - std::stringstream ss; - std::string geometry; + if (!waycache.count(mit->ref)) { + noWay = true; + break; + } + auto way = waycache.at(mit->ref); + + // When Relation is MultiPolygon but way's geometry is LineString + // append LineString to points to convert it later to a Polygon + if (using_linestring_tmp || (isMultiPolygon && + boost::geometry::num_points(way->linestring) > 0 && + boost::geometry::num_points(way->polygon) == 0) + ) { + + // MultiPolygon made of multiple LineStrings + + using_linestring_tmp = true; + std::stringstream ss; + std::string geometry; + if (boost::geometry::num_points(way->polygon) > 0) { + ss << std::setprecision(12) << boost::geometry::wkt(way->polygon.outer()); + geometry = ss.str(); + geometry.erase(geometry.size() - 1); + } else { + + if (!first) { + // Check if linestrings needs to be reversed + bool reverse_line = bg::equals(lastLinestring.front(), way->linestring.front()) || + bg::equals(lastLinestring.front(), way->linestring.back()) || + bg::equals(lastLinestring.back(), way->linestring.front()) || + bg::equals(lastLinestring.back(), way->linestring.back()); + if (reverse_line) { + bg::reverse(way->linestring); + } + } + lastLinestring = way->linestring; + + ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); + geometry = ss.str(); + } + geometry.erase(0,11); + geometry.erase(geometry.size() - 1); + linestring_tmp += "(" + geometry + "),"; - if (isMultiPolygon) { - ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); - geometry = ss.str(); - geometry.erase(0,8); - } else { - ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); - geometry = ss.str(); - geometry.erase(0,11); - } - - geometry.erase(geometry.size() - 1); - if (first && (mit->role == "outer")) { - geometry_str += geometry + ","; - } else { - if (mit->role == "inner") { - innerGeoStr += geometry + ","; } else { - geometry_str += geometry + ","; - geometry_str += innerGeoStr; - innerGeoStr = ""; + + // MultiPolygon or MultiLineString + + // When Relation is MultiLineString but way's geometry is Polygon + if (isMultiLineString && boost::geometry::num_points(way->linestring) == 0 && + boost::geometry::num_points(way->polygon) > 0 + ) { + // Convert way's Polygon to LineString + bg::assign_points(way->linestring, way->polygon.outer()); + } + + std::stringstream ss; + std::string geometry; + + if (isMultiPolygon) { + ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); + geometry = ss.str(); + // Erase "POLYGON(" + geometry.erase(0,8); + } else { + ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); + geometry = ss.str(); + // Erase "LINESTRING(" + geometry.erase(0,11); + } + + // Erase ")" + geometry.erase(geometry.size() - 1); + + if (isMultiLineString) { + geometry = "(" + geometry + ")"; + } + + if (first && (mit->role != "inner")) { + first = false; + geometry_str += geometry + ","; + } else { + if (mit->role == "inner" && isMultiPolygon) { + innerGeoStr += geometry + ","; + } else { + geometry_str += geometry + ","; + geometry_str += innerGeoStr; + innerGeoStr = ""; + } + } } } } - if (innerGeoStr.size() > 0) { - geometry_str += innerGeoStr; - } - geometry_str.erase(geometry_str.size() - 1); - geometry_str += "))"; if (!noWay) { + if (linestring_tmp.size() == 0) { + if (isMultiLineString) { + geometry_str.erase(geometry_str.size() - 1); + geometry_str = "MULTILINESTRING(" + geometry_str; + } else { + geometry_str = "MULTIPOLYGON((" + geometry_str; + } + if (innerGeoStr.size() > 0) { + geometry_str += innerGeoStr; + } + geometry_str.erase(geometry_str.size() - 1); + geometry_str += "))"; + } else { + linestring_tmp.erase(linestring_tmp.size() - 1); + geometry_str = "MULTILINESTRING(" + linestring_tmp + ")"; + + boost::geometry::read_wkt(geometry_str, relation.multilinestring); + linestring_t mergedLineString; + for (auto& line : relation.multilinestring) { + bg::append(mergedLineString, line); + } + // Create a polygon from the linestring + polygon_t polygon; + bg::append(polygon.outer(), mergedLineString); + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(polygon); + geometry_str = ss.str(); + // Erase "POLYGON" + geometry_str.erase(0, 8); + geometry_str = "MULTIPOLYGON((" + geometry_str + ")"; + + } + if (isMultiPolygon) { boost::geometry::read_wkt(geometry_str, relation.multipolygon); } else { boost::geometry::read_wkt(geometry_str, relation.multilinestring); } } + } } @@ -535,10 +620,14 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) for (auto it = std::begin(changes); it != std::end(changes); it++) { OsmChange *change = it->get(); + bool debug = false; // Filter nodes for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { OsmNode *node = nit->get(); + if (node->id == 1508976013) { + debug = true; + } if (poly.empty() || boost::geometry::within(node->point, poly)) { node->priority = true; nodecache[node->id] = node->point; @@ -557,10 +646,6 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { if (nodecache.count(*rit) && boost::geometry::within(nodecache[*rit], poly)) { way->priority = true; - if (boost::geometry::within(nodecache[*rit], poly)) { - std::stringstream ss; - ss << std::setprecision(12) << boost::geometry::wkt(nodecache[*rit]); - } break; } } @@ -581,10 +666,13 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) for (auto mit = std::begin(relation->members); mit != std::end(relation->members); ++mit) { if (waycache.count(mit->ref)) { auto way = waycache.at(mit->ref); - if (way->priority) { - relation->priority = true; + if (!way->priority) { + relation->priority = false; break; } + } else { + relation->priority = false; + break; } } } diff --git a/src/osm/osmobjects.cc b/src/osm/osmobjects.cc index d1f93fdd7..e87ffb68f 100644 --- a/src/osm/osmobjects.cc +++ b/src/osm/osmobjects.cc @@ -134,7 +134,13 @@ OsmRelationMember::dump() const { std::cerr << "\t\tDumping Relation member" << std::endl; std::cerr << "\t\t\tRef: " << ref << std::endl; - std::cerr << "\t\t\type: " << type << std::endl; + if (type == node) { + std::cerr << "\t\t\tType: OsmNode" << std::endl; + } else if (type == way) { + std::cerr << "\t\t\tType: OsmWay" << std::endl; + } else if (type == relation) { + std::cerr << "\t\t\tType: OsmRelation" << std::endl; + } std::cerr << "\t\t\tRole: " << role << std::endl; } diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index 9a758f19a..468c8a19a 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -235,8 +235,7 @@ class OsmRelation : public OsmObject { bool isMultiPolygon(void) const { return (tags.count("type") && - (tags.at("type") == "multipolygon" || - tags.at("type") == "boundary") + (tags.at("type") == "multipolygon") ); }; @@ -244,7 +243,7 @@ class OsmRelation : public OsmObject { bool isMultiLineString(void) const { return (tags.count("type") && - tags.at("type") == "multilinestring" + (tags.at("type") != "multipolygon") ); }; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index ee8740f2f..915eed94b 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -89,11 +89,12 @@ QueryRaw::buildTagsQuery(std::map tags) const { std::string buildMembersQuery(std::list members) { if (members.size() > 0) { - std::string membersStr = "jsonb_build_array("; + std::string membersStr = "'["; int count = 0; for (auto mit = std::begin(members); mit != std::end(members); ++mit) { - membersStr += "jsonb_build_object("; - std::string member_format = "'%s', '%s',"; + count++; + membersStr += "{"; + std::string member_format = "\"%s\": \"%s\","; boost::format member_fmt(member_format); member_fmt % "role"; member_fmt % mit->role; @@ -101,14 +102,13 @@ buildMembersQuery(std::list members) { member_fmt % "type"; member_fmt % mit->type; membersStr += member_fmt.str(); - member_fmt % "ref"; - member_fmt % mit->ref; - membersStr += member_fmt.str(); - membersStr.erase(membersStr.size() - 1); - membersStr += "),"; + membersStr += "\"ref\":"; + membersStr += std::to_string(mit->ref); + membersStr += "},"; } membersStr.erase(membersStr.size() - 1); - return membersStr += ")"; + + return membersStr += "]'"; } else { return "null"; } @@ -312,6 +312,15 @@ QueryRaw::applyChange(const OsmRelation &relation) const if (relation.action == osmobjects::create || relation.action == osmobjects::modify) { + // if (relation.id == 3365911) { + if (geostring == "MULTILINESTRING()") { + std::cout << "(0) HERE! Relation MULTILINESTRING() " << relation.id << std::endl; + } + if (geostring == "MULTIPOLYGON()") { + std::cout << "(0) HERE! Relation MULTIPOLYGON() " << relation.id << std::endl; + } + // } + query = "INSERT INTO relations as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; @@ -331,6 +340,7 @@ QueryRaw::applyChange(const OsmRelation &relation) const // geometry std::string geometry; geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; + std::cout << geostring << std::endl; fmt % geometry; // timestamp @@ -398,9 +408,17 @@ QueryRaw::getRelationsByWaysRefs(std::string &wayIds) const rel->id = (*rel_it)[0].as(); std::string refs_str = (*rel_it)[1].as(); auto members = parseJSONArrayStr(refs_str); + for (auto mit = members.begin(); mit != members.end(); ++mit) { - rel->addMember(std::stol(mit->at("ref")), osmobjects::osmtype_t::way, mit->at("role")); + auto memberType = osmobjects::osmtype_t::way; + if (mit->at("type") == "n") { + memberType = osmobjects::osmtype_t::node; + } else if (mit->at("type") == "r") { + memberType = osmobjects::osmtype_t::relation; + } + rel->addMember(std::stol(mit->at("ref")), memberType, mit->at("role")); } + rel->version = (*rel_it)[2].as(); auto tags = (*rel_it)[3]; if (!tags.is_null()) { @@ -429,14 +447,20 @@ QueryRaw::getWaysByIds(std::string &waysIds, std::mapquery(waysQuery); // Fill vector of OsmWay objects for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { auto way = std::make_shared(); + auto type = (*way_it)[2].as(); way->id = (*way_it)[0].as(); - boost::geometry::read_wkt((*way_it)[1].as(), way->polygon); + if (type == "polygon") { + boost::geometry::read_wkt((*way_it)[1].as(), way->polygon); + } else { + boost::geometry::read_wkt((*way_it)[1].as(), way->linestring); + } waycache.insert(std::pair(way->id, std::make_shared(*way))); } } @@ -528,7 +552,6 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const if (std::find(removedRelations.begin(), removedRelations.end(), relation->id) == removedRelations.end()) { relation->action = osmobjects::modify; change->relations.push_back(relation); - } } osmchanges->changes.push_back(change); @@ -561,6 +584,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const boost::geometry::append(way->linestring, osmchanges->nodecache.at(*rit)); } } + if (way->isClosed()) { way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; } @@ -577,6 +601,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const // Filter out all relations that doesn't have at least 1 way in cache std::string relsForWayCacheIds; + bool debug = false; for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { OsmChange *change = it->get(); for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { @@ -605,7 +630,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const // Get all missing ways geometries for relations if (relsForWayCacheIds != "") { relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); - getWaysByIds(relsForWayCacheIds, osmchanges->waycache); + getWaysByIds(relsForWayCacheIds, osmchanges->waycache); } // Build relation geometries @@ -845,13 +870,17 @@ QueryRaw::getRelationsFromDB(long lastid, int pageSize) { if (!refs.is_null()) { auto refs = parseJSONArrayStr((*rel_it)[1].as()); for (auto ref_it = refs.begin(); ref_it != refs.end(); ++ref_it) { - if (ref_it->at("type") == "w" && (ref_it->at("role") == "inner" || ref_it->at("role") == "outer")) { - relation.addMember( - std::stoi(ref_it->at("ref")), - osmobjects::osmtype_t::way, - ref_it->at("role") - ); + auto relType = osmobjects::osmtype_t::way; + if (ref_it->at("type") == "n") { + relType = osmobjects::osmtype_t::node; + } else if (ref_it->at("type") == "r") { + relType = osmobjects::osmtype_t::relation; } + relation.addMember( + std::stol(ref_it->at("ref")), + relType, + ref_it->at("role") + ); } std::string geometry = (*rel_it)[2].as(); if (geometry.substr(0, 12) == "MULTIPOLYGON") { diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 9d1e8aa1a..0e64f45bf 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -548,7 +548,6 @@ threadOsmChange(OsmChangeTask osmChangeTask) if (relation->action != osmobjects::remove && !relation->priority) { continue; } - // Remove deleted relations from validation table if (!config->disable_validation && relation->action == osmobjects::remove) { removed_relations->push_back(relation->id); @@ -556,7 +555,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) // Update relations, ignore new ones outside priority area if (!config->disable_raw) { - // task.query += queryraw->applyChange(*relation); + task.query += queryraw->applyChange(*relation); } } diff --git a/src/testsuite/libunderpass.all/areafilter-test.cc b/src/testsuite/libunderpass.all/areafilter-test.cc index d64ef63e7..18a456141 100644 --- a/src/testsuite/libunderpass.all/areafilter-test.cc +++ b/src/testsuite/libunderpass.all/areafilter-test.cc @@ -57,9 +57,6 @@ countFeatures(TestOsmChange &osmchange) { } } } - // std::cout << "nodeCount: " << nodeCount << std::endl; - // std::cout << "wayCount: " << wayCount << std::endl; - // std::cout << "relCount: " << relCount << std::endl; return nodeCount + wayCount + relCount; } diff --git a/src/testsuite/libunderpass.all/raw-test.cc b/src/testsuite/libunderpass.all/raw-test.cc index 691b0e4e2..1eb9eb834 100644 --- a/src/testsuite/libunderpass.all/raw-test.cc +++ b/src/testsuite/libunderpass.all/raw-test.cc @@ -82,6 +82,7 @@ bool processFile(const std::string &filename, std::shared_ptr &db) { multipolygon_t poly; osmchanges->readChanges(destdir_base + "/testsuite/testdata/raw/" + filename); queryraw->buildGeometries(osmchanges, poly); + osmchanges->areaFilter(poly); std::string rawquery; for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); ++it) { @@ -105,13 +106,14 @@ bool processFile(const std::string &filename, std::shared_ptr &db) { db->query(rawquery); } -const std::map expectedGeometries = { - {101874, "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"}, - {101875, "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"}, - {101875-2, "POLYGON((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953))"}, - {211766, "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))"}, - {211766-2, "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.7260807753 4.62037032501,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))"}, - {211776, "MULTILINESTRING((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"} +const std::vector expectedGeometries = { + "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))", + "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))", + "POLYGON((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953))", + "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))", + "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.7260807753 4.62037032501,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))", + "MULTILINESTRING((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836),(21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))", + "MULTIPOLYGON(((-59.8355946 -36.7448782,-59.8353203 -36.7452085,-59.8368532 -36.7480926,-59.8370667 -36.7483355,-59.8408639 -36.7515154,-59.8482593 -36.745907,-59.8486299 -36.7452032,-59.848532 -36.7453269,-59.8483958 -36.7456167,-59.8482593 -36.745907,-59.8486299 -36.7452032,-59.8486213 -36.7451521,-59.8481455 -36.7432981,-59.8478834 -36.7425679,-59.8478326 -36.7423963,-59.8478326 -36.7423963,-59.8477902 -36.7422529,-59.8477764 -36.7422007,-59.8477733 -36.74215,-59.8477773 -36.742095,-59.8477972 -36.7420329,-59.8478247 -36.7419783,-59.8478609 -36.7419236,-59.8479126 -36.7418808,-59.84795 -36.7418607,-59.8480462 -36.7418147,-59.8480692 -36.7417819,-59.8480709 -36.7417548,-59.8480673 -36.7416893,-59.8479835 -36.741163,-59.8479787 -36.7411211,-59.8479562 -36.7410819,-59.847923 -36.7410414,-59.8478722 -36.7410094,-59.8475273 -36.7408127,-59.8474352 -36.7407496,-59.8473715 -36.7406912,-59.8473254 -36.7406329,-59.8472829 -36.7405588,-59.8472494 -36.7404962,-59.8472589 -36.7404251,-59.8472978 -36.7403582,-59.8473424 -36.74032,-59.8473864 -36.7402921,-59.8474375 -36.740274,-59.8474922 -36.740256,-59.8475468 -36.7402465,-59.8476241 -36.7402304,-59.8476784 -36.7402171,-59.8477563 -36.7401919,-59.8478265 -36.7401545,-59.8479128 -36.7400779,-59.8479937 -36.7399885,-59.8480801 -36.7398796,-59.8481504 -36.7397665,-59.848411 -36.7391216,-59.8484023 -36.739028,-59.8483557 -36.7389343,-59.8482783 -36.7388702,-59.84822 -36.7388242,-59.8481308 -36.7388079,-59.8480003 -36.7387583,-59.8478575 -36.7386841,-59.8477933 -36.738618,-59.8477365 -36.7385158,-59.8477106 -36.7384235,-59.8477053 -36.7382963,-59.8477134 -36.7381998,-59.8477433 -36.7381126,-59.8478321 -36.738022,-59.8479105 -36.7379644,-59.8480011 -36.7379216,-59.8482127 -36.7378122,-59.8482877 -36.7377698,-59.8483566 -36.7377126,-59.848393 -36.737632,-59.8484294 -36.7375366,-59.8485761 -36.7372026,-59.848605 -36.7370773,-59.8486246 -36.7369372,-59.8486196 -36.7368366,-59.8485807 -36.7367015,-59.848529 -36.7365836,-59.8484717 -36.7364835,-59.8483887 -36.7363497,-59.8477548 -36.7356502,-59.8477339 -36.7356248,-59.8477339 -36.7356248,-59.8475634 -36.7357007,-59.8474292 -36.7357691,-59.8473073 -36.7358571,-59.8469617 -36.7361243,-59.8447338 -36.737825,-59.8424572 -36.7395354,-59.8423067 -36.7396527,-59.8386641 -36.7424968,-59.838225 -36.7428388,-59.8355946 -36.7448782)))" }; std::string @@ -123,7 +125,7 @@ getWKT(const polygon_t &polygon) { std::string getWKTFromDB(const std::string &table, const long id, std::shared_ptr &db) { - auto result = db->query("SELECT ST_AsText(geom, 4326) from relations where osm_id=" + std::to_string(id)); + auto result = db->query("SELECT ST_AsText(geom, 4326), refs from relations where osm_id=" + std::to_string(id)); for (auto r_it = result.begin(); r_it != result.end(); ++r_it) { return (*r_it)[0].as(); } @@ -148,20 +150,15 @@ main(int argc, char *argv[]) if (db->connect(dbconn + " dbname=underpass_test")) { auto queryraw = std::make_shared(db); std::map> waycache; - std::string waysIds; processFile("raw-case-1.osc", db); processFile("raw-case-2.osc", db); - for (auto const& x : expectedGeometries) { - waysIds += std::to_string(x.first) + ","; - } - waysIds.erase(waysIds.size() - 1); - + std::string waysIds = "101874,101875"; queryraw->getWaysByIds(waysIds, waycache); // 4 created Nodes, 1 created Way (same changeset) - if ( getWKT(waycache.at(101874)->polygon).compare(expectedGeometries.at(101874)) == 0) { + if ( getWKT(waycache.at(101874)->polygon).compare(expectedGeometries[0]) == 0) { runtest.pass("4 created Nodes, 1 created Ways (same changeset)"); } else { runtest.fail("4 created Nodes, 1 created Ways (same changeset)"); @@ -169,7 +166,7 @@ main(int argc, char *argv[]) } // 1 created Way, 4 existing Nodes (different changeset) - if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries.at(101875)) == 0) { + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[1]) == 0) { runtest.pass("1 created Way, 4 existing Nodes (different changesets)"); } else { runtest.fail("1 created Way, 4 existing Nodes (different changesets)"); @@ -180,8 +177,7 @@ main(int argc, char *argv[]) processFile("raw-case-3.osc", db); waycache.erase(101875); queryraw->getWaysByIds(waysIds, waycache); - - if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries.at(101875-2)) == 0) { + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[2]) == 0) { runtest.pass("1 modified Node, indirectly modify other existing Ways (different changesets)"); } else { runtest.fail("1 modified Node, indirectly modify other existing Ways (different changesets)"); @@ -190,7 +186,7 @@ main(int argc, char *argv[]) // 1 created Relation referencing 1 created Way and 1 existing Way processFile("raw-case-4.osc", db); - if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries.at(211766)) == 0) { + if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[3]) == 0) { runtest.pass("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); } else { runtest.fail("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); @@ -199,8 +195,7 @@ main(int argc, char *argv[]) // 1 modified Node, indirectly modify other existing Ways and 1 Relation processFile("raw-case-5.osc", db); - - if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries.at(211766-2)) == 0) { + if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[4]) == 0) { runtest.pass("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); } else { runtest.fail("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); @@ -209,13 +204,23 @@ main(int argc, char *argv[]) // 4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring processFile("raw-case-6.osc", db); - if ( getWKTFromDB("relations", 211776, db).compare(expectedGeometries.at(211776)) == 0) { + if ( getWKTFromDB("relations", 211776, db).compare(expectedGeometries[5]) == 0) { runtest.pass("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); } else { runtest.fail("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); return 1; } + // Complex, 1 polygon relation made of multiple ways + processFile("raw-case-7.osc", db); + if ( getWKTFromDB("relations", 17331328, db).compare(expectedGeometries[6]) == 0) { + runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset)"); + } else { + runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset)"); + return 1; + } + + } diff --git a/src/testsuite/testdata/raw/raw-case-6.osc b/src/testsuite/testdata/raw/raw-case-6.osc index 948188d17..e7baaa44a 100644 --- a/src/testsuite/testdata/raw/raw-case-6.osc +++ b/src/testsuite/testdata/raw/raw-case-6.osc @@ -8,6 +8,7 @@ + diff --git a/src/testsuite/testdata/raw/raw-case-7.osc b/src/testsuite/testdata/raw/raw-case-7.osc new file mode 100644 index 000000000..261b8228b --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-7.osc @@ -0,0 +1,229 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/underpass.cc b/src/underpass.cc index 6026c6d62..d5082ddf7 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -297,7 +297,9 @@ main(int argc, char *argv[]) osmboundary = &geou.boundary; } osmchange->destdir_base = config.destdir_base; - osmchange->dump(); + if (!config.silent) { + osmchange->dump(); + } osmChangeThread = std::thread(replicatorthreads::startMonitorChanges, std::ref(osmchange), std::ref(*osmboundary), config); } From 0b0da0fd1d27dddfb3d1b4aaa28252f7a3c8ad73 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 20 Mar 2024 09:53:01 -0300 Subject: [PATCH 3/9] Working on Relations/MultiPolygon/MultiLineString support --- src/osm/osmchange.cc | 41 +++--- src/raw/queryraw.cc | 126 +++++++++--------- .../libunderpass.all/areafilter-test.cc | 64 +++++---- src/testsuite/libunderpass.all/raw-test.cc | 9 ++ src/testsuite/testdata/areafilter-test.osm | 37 +---- 5 files changed, 141 insertions(+), 136 deletions(-) diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index 5c46ca96c..d7d8ad42a 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -128,6 +128,7 @@ OsmChangeFile::readChanges(const std::string &file) return true; } +// Used for testing void OsmChangeFile::buildGeometriesFromNodeCache() { for (auto it = std::begin(changes); it != std::end(changes); ++it) { @@ -140,6 +141,12 @@ OsmChangeFile::buildGeometriesFromNodeCache() { if (way->isClosed()) { way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; } + std::stringstream ss; + if (way->isClosed()) { + ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); + } else { + ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); + } waycache.insert(std::pair(way->id, std::make_shared(*way))); } for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { @@ -149,23 +156,28 @@ OsmChangeFile::buildGeometriesFromNodeCache() { } } +// FIXME void OsmChangeFile::buildRelationGeometry(osmobjects::OsmRelation &relation) { bool first = true; bool isMultiPolygon = relation.isMultiPolygon(); bool isMultiLineString = relation.isMultiLineString(); + if (isMultiPolygon || isMultiLineString) { std::string innerGeoStr; std::string geometry_str; + // A way is missing from cache bool noWay = false; + // There are not Way relation members + bool noWayMembers = true; std::string linestring_tmp; bool using_linestring_tmp = false; linestring_t lastLinestring; for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { if (mit->type == osmobjects::way) { - + noWayMembers = false; if (!waycache.count(mit->ref)) { noWay = true; break; @@ -192,12 +204,14 @@ OsmChangeFile::buildRelationGeometry(osmobjects::OsmRelation &relation) { if (!first) { // Check if linestrings needs to be reversed - bool reverse_line = bg::equals(lastLinestring.front(), way->linestring.front()) || - bg::equals(lastLinestring.front(), way->linestring.back()) || - bg::equals(lastLinestring.back(), way->linestring.front()) || - bg::equals(lastLinestring.back(), way->linestring.back()); - if (reverse_line) { - bg::reverse(way->linestring); + if (lastLinestring.size() > 0 && way->linestring.size() > 0) { + bool reverse_line = bg::equals(lastLinestring.front(), way->linestring.front()) || + bg::equals(lastLinestring.front(), way->linestring.back()) || + bg::equals(lastLinestring.back(), way->linestring.front()) || + bg::equals(lastLinestring.back(), way->linestring.back()); + if (reverse_line) { + bg::reverse(way->linestring); + } } } lastLinestring = way->linestring; @@ -259,7 +273,7 @@ OsmChangeFile::buildRelationGeometry(osmobjects::OsmRelation &relation) { } } - if (!noWay) { + if (!noWay && !noWayMembers) { if (linestring_tmp.size() == 0) { if (isMultiLineString) { geometry_str.erase(geometry_str.size() - 1); @@ -298,8 +312,8 @@ OsmChangeFile::buildRelationGeometry(osmobjects::OsmRelation &relation) { } else { boost::geometry::read_wkt(geometry_str, relation.multilinestring); } - } + } } } @@ -625,9 +639,6 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) // Filter nodes for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { OsmNode *node = nit->get(); - if (node->id == 1508976013) { - debug = true; - } if (poly.empty() || boost::geometry::within(node->point, poly)) { node->priority = true; nodecache[node->id] = node->point; @@ -659,10 +670,8 @@ OsmChangeFile::areaFilter(const multipolygon_t &poly) // Filter relations for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { OsmRelation *relation = rit->get(); - if (poly.empty()) { - relation->priority = true; - } else { - relation->priority = false; + relation->priority = true; + if (!poly.empty()) { for (auto mit = std::begin(relation->members); mit != std::end(relation->members); ++mit) { if (waycache.count(mit->ref)) { auto way = waycache.at(mit->ref); diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 915eed94b..127fadba3 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -301,82 +301,81 @@ std::string QueryRaw::applyChange(const OsmRelation &relation) const { std::string query = ""; - std::stringstream ss; - if (relation.isMultiPolygon()) { - ss << std::setprecision(12) << boost::geometry::wkt(relation.multipolygon); - } else { - ss << std::setprecision(12) << boost::geometry::wkt(relation.multilinestring); - } - - std::string geostring = ss.str(); if (relation.action == osmobjects::create || relation.action == osmobjects::modify) { - // if (relation.id == 3365911) { - if (geostring == "MULTILINESTRING()") { - std::cout << "(0) HERE! Relation MULTILINESTRING() " << relation.id << std::endl; - } - if (geostring == "MULTIPOLYGON()") { - std::cout << "(0) HERE! Relation MULTIPOLYGON() " << relation.id << std::endl; + std::stringstream ss; + if (relation.isMultiPolygon()) { + ss << std::setprecision(12) << boost::geometry::wkt(relation.multipolygon); + } else { + ss << std::setprecision(12) << boost::geometry::wkt(relation.multilinestring); } - // } + std::string geostring = ss.str(); - query = "INSERT INTO relations as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; - std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ - ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; - boost::format fmt(format); + // Ignore empty geometries + if (geostring != "MULTILINESTRING()" && geostring != "MULTIPOLYGON()") { - // osm_id - fmt % relation.id; + query = "INSERT INTO relations as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ + ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; + boost::format fmt(format); - // tags - auto tags = buildTagsQuery(relation.tags); - fmt % tags; + // osm_id + fmt % relation.id; - // refs - auto refs = buildMembersQuery(relation.members); - fmt % refs; + // tags + auto tags = buildTagsQuery(relation.tags); + fmt % tags; - // geometry - std::string geometry; - geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; - std::cout << geostring << std::endl; - fmt % geometry; + // refs + auto refs = buildMembersQuery(relation.members); + fmt % refs; - // timestamp - std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); - fmt % timestamp; - // version - fmt % relation.version; - // user - fmt % dbconn->escapedString(relation.user); - // uid - fmt % relation.uid; - // changeset - fmt % relation.changeset; + // geometry + std::string geometry; + geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; + + std::cout << "Relation " << relation.id << " " << geostring << std::endl; + + fmt % geometry; - // ON CONFLICT - fmt % tags; - fmt % refs; - fmt % geometry; - fmt % timestamp; - fmt % relation.version; - fmt % dbconn->escapedString(relation.user); - fmt % relation.uid; - fmt % relation.changeset; - fmt % relation.version; + // timestamp + std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); + fmt % timestamp; + // version + fmt % relation.version; + // user + fmt % dbconn->escapedString(relation.user); + // uid + fmt % relation.uid; + // changeset + fmt % relation.changeset; - query += fmt.str(); + // ON CONFLICT + fmt % tags; + fmt % refs; + fmt % geometry; + fmt % timestamp; + fmt % relation.version; + fmt % dbconn->escapedString(relation.user); + fmt % relation.uid; + fmt % relation.changeset; + fmt % relation.version; - for (auto it = std::begin(relation.members); it != std::end(relation.members); ++it) { - query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(it->ref) + ");"; - } + query += fmt.str(); + for (auto it = std::begin(relation.members); it != std::end(relation.members); ++it) { + query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(it->ref) + ");"; + } + } else { + std::cout << "Relation " << relation.id << " geometry is empty" << std::endl; + } } else if (relation.action == osmobjects::remove) { query += "DELETE FROM relations where osm_id = " + std::to_string(relation.id) + ";"; } - return query;} + return query; +} std::vector arrayStrToVector(std::string &refs_str) { refs_str.erase(0, 1); @@ -599,7 +598,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } } - // Filter out all relations that doesn't have at least 1 way in cache + // Relations std::string relsForWayCacheIds; bool debug = false; for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { @@ -615,14 +614,11 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } } if (getWaysForRelation) { - relation->priority = true; for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { if (!osmchanges->waycache.count(mit->ref)) { relsForWayCacheIds += std::to_string(mit->ref) + ","; } } - } else { - relation->priority = false; } } } @@ -630,7 +626,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const // Get all missing ways geometries for relations if (relsForWayCacheIds != "") { relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); - getWaysByIds(relsForWayCacheIds, osmchanges->waycache); + getWaysByIds(relsForWayCacheIds, osmchanges->waycache); } // Build relation geometries @@ -638,9 +634,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const OsmChange *change = it->get(); for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { OsmRelation *relation = rel_it->get(); - if (relation->priority) { - osmchanges->buildRelationGeometry(*relation); - } + osmchanges->buildRelationGeometry(*relation); } } } diff --git a/src/testsuite/libunderpass.all/areafilter-test.cc b/src/testsuite/libunderpass.all/areafilter-test.cc index 18a456141..1b5de7a79 100644 --- a/src/testsuite/libunderpass.all/areafilter-test.cc +++ b/src/testsuite/libunderpass.all/areafilter-test.cc @@ -127,6 +127,7 @@ main(int argc, char *argv[]) } // ChangeSet - Small area in North Africa + // outside, not in priority area changeset.readChanges(changesetFile); changeset.areaFilter(polySmallArea); testChangeset = changeset.changes.front().get(); @@ -137,6 +138,7 @@ main(int argc, char *argv[]) } // ChangeSet - Empty polygon + // inside, in priority area changeset.readChanges(changesetFile); changeset.areaFilter(polyEmpty); testChangeset = changeset.changes.front().get(); @@ -150,6 +152,7 @@ main(int argc, char *argv[]) // ChangeSet - Half area changeset.readChanges(changesetFile); changeset.areaFilter(polyHalf); + // inside, in priority area testChangeset = changeset.changes.front().get(); if (testChangeset && testChangeset->priority) { runtest.pass("ChangeSet areaFilter - true (half area)"); @@ -160,49 +163,64 @@ main(int argc, char *argv[]) // -- OsmChanges - // OsmChange - Empty poly + // // OsmChange - Empty poly osmchange.readChanges(osmchangeFile); - osmchange.areaFilter(polyEmpty); - if (getPriority(osmchange) && countFeatures(osmchange) == 62) { - runtest.pass("OsmChange areaFilter - true (Empty poly)"); - } else { - runtest.fail("OsmChange areaFilter - true (Empty poly)"); - return 1; - } - - // OsmChange - Whole world osmchange.buildGeometriesFromNodeCache(); - osmchange.areaFilter(polyWholeWorld); - if (getPriority(osmchange) && countFeatures(osmchange) == 62) { - runtest.pass("OsmChange areaFilter - true (whole world)"); + + // osmchange.areaFilter(polyEmpty); + // if (getPriority(osmchange) && countFeatures(osmchange) == 62) { + // runtest.pass("OsmChange areaFilter - true (Empty poly)"); + // } else { + // runtest.fail("OsmChange areaFilter - true (Empty poly)"); + // return 1; + // } + + // // OsmChange - Whole world + // osmchange.areaFilter(polyWholeWorld); + // if (getPriority(osmchange) && countFeatures(osmchange) == 62) { + // runtest.pass("OsmChange areaFilter - true (whole world)"); + // } else { + // runtest.fail("OsmChange areaFilter - true (whole world)"); + // return 1; + // } + + // OsmChange - Small area in North Africa + // Outside priority area, count should be 0 + // clearChanges(osmchange); + // osmchange.readChanges(osmchangeFile); + osmchange.areaFilter(polySmallArea); + if (countFeatures(osmchange) == 0) { + runtest.pass("OsmChange areaFilter - true (0)"); } else { - runtest.fail("OsmChange areaFilter - true (whole world)"); + runtest.fail("OsmChange areaFilter - true (0)"); return 1; } - // OsmChange - Small area + // OsmChange - Small area in Bangladesh + // 28 nodes / 5 ways / 1 relation inside priority area, count should be 34 clearChanges(osmchange); osmchange.readChanges(osmchangeFile); + osmchange.buildGeometriesFromNodeCache(); osmchange.areaFilter(polyHalf); - if (countFeatures(osmchange) == 35) { - runtest.pass("OsmChange areaFilter - true (small area)"); + if (countFeatures(osmchange) == 34) { + runtest.pass("OsmChange areaFilter - true (34)"); } else { - runtest.fail("OsmChange areaFilter - true (small area)"); + runtest.fail("OsmChange areaFilter - true (34)"); return 1; } - // OsmChange - Smaller area + // OsmChange - Smaller area in Bangladesh + // 12 nodes / 3 ways / 1 relation inside priority area, count should be 16 clearChanges(osmchange); osmchange.readChanges(osmchangeFile); osmchange.areaFilter(polyHalfSmall); - if (countFeatures(osmchange) == 17) { - runtest.pass("OsmChange areaFilter - true (smaller area)"); + if (countFeatures(osmchange) == 16) { + runtest.pass("OsmChange areaFilter - true (smaller area inside)"); } else { - runtest.fail("OsmChange areaFilter - true (smaller area)"); + runtest.fail("OsmChange areaFilter - true (smaller area inside)"); return 1; } - } // local Variables: diff --git a/src/testsuite/libunderpass.all/raw-test.cc b/src/testsuite/libunderpass.all/raw-test.cc index 1eb9eb834..ba2032156 100644 --- a/src/testsuite/libunderpass.all/raw-test.cc +++ b/src/testsuite/libunderpass.all/raw-test.cc @@ -220,6 +220,15 @@ main(int argc, char *argv[]) return 1; } + // + processFile("raw-case-8.osc", db); + // if ( getWKTFromDB("relations", 17331328, db).compare(expectedGeometries[6]) == 0) { + // runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset)"); + // } else { + // runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset)"); + // return 1; + // } + } diff --git a/src/testsuite/testdata/areafilter-test.osm b/src/testsuite/testdata/areafilter-test.osm index f0873cbd2..9be932647 100644 --- a/src/testsuite/testdata/areafilter-test.osm +++ b/src/testsuite/testdata/areafilter-test.osm @@ -41,6 +41,10 @@ + + + + @@ -113,47 +117,18 @@ - - - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 8c8026a3dffb5ad73341f8cf310ea1dd65251efc Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Thu, 21 Mar 2024 10:26:54 -0300 Subject: [PATCH 4/9] Working on relations support --- src/osm/osmchange.cc | 328 +++++++++------ src/osm/osmobjects.hh | 8 - src/raw/queryraw.cc | 108 ++--- src/replicator/threads.cc | 34 +- .../libunderpass.all/areafilter-test.cc | 58 ++- src/testsuite/libunderpass.all/raw-test.cc | 136 ++++--- src/testsuite/testdata/raw/raw-case-7.osc | 18 +- src/testsuite/testdata/raw/raw-case-8.osc | 198 +++++++++ src/testsuite/testdata/raw/raw-case-9.osc | 380 ++++++++++++++++++ 9 files changed, 953 insertions(+), 315 deletions(-) create mode 100644 src/testsuite/testdata/raw/raw-case-8.osc create mode 100644 src/testsuite/testdata/raw/raw-case-9.osc diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index d7d8ad42a..76c459b5c 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -157,163 +157,239 @@ OsmChangeFile::buildGeometriesFromNodeCache() { } // FIXME +// TODO: refactor, divide this function into multiple parts void OsmChangeFile::buildRelationGeometry(osmobjects::OsmRelation &relation) { + + // A way is missing from cache + bool noWay = false; + // There are not Way relation members + bool noWayMembers = true; + bool first = true; + std::string innerGeoStr; + std::string geometry_str; + std::string linestring_tmp; + bool using_linestring_tmp = false; + linestring_t lastLinestring; + point_t firstLinestringPoint; bool isMultiPolygon = relation.isMultiPolygon(); - bool isMultiLineString = relation.isMultiLineString(); - - if (isMultiPolygon || isMultiLineString) { - std::string innerGeoStr; - std::string geometry_str; - - // A way is missing from cache - bool noWay = false; - // There are not Way relation members - bool noWayMembers = true; - std::string linestring_tmp; - bool using_linestring_tmp = false; - linestring_t lastLinestring; - - for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { - if (mit->type == osmobjects::way) { - noWayMembers = false; - if (!waycache.count(mit->ref)) { - noWay = true; - break; - } - auto way = waycache.at(mit->ref); + bool close = false; - // When Relation is MultiPolygon but way's geometry is LineString - // append LineString to points to convert it later to a Polygon - if (using_linestring_tmp || (isMultiPolygon && - boost::geometry::num_points(way->linestring) > 0 && - boost::geometry::num_points(way->polygon) == 0) - ) { + for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { - // MultiPolygon made of multiple LineStrings - - using_linestring_tmp = true; - std::stringstream ss; - std::string geometry; - if (boost::geometry::num_points(way->polygon) > 0) { - ss << std::setprecision(12) << boost::geometry::wkt(way->polygon.outer()); - geometry = ss.str(); - geometry.erase(geometry.size() - 1); - } else { + // Process Way objects only, not Nodes or other Relations + if (mit->type == osmobjects::way) { + noWayMembers = false; + + if (close == true) { + first = true; + close = false; + using_linestring_tmp = false; + } - if (!first) { - // Check if linestrings needs to be reversed - if (lastLinestring.size() > 0 && way->linestring.size() > 0) { - bool reverse_line = bg::equals(lastLinestring.front(), way->linestring.front()) || - bg::equals(lastLinestring.front(), way->linestring.back()) || - bg::equals(lastLinestring.back(), way->linestring.front()) || - bg::equals(lastLinestring.back(), way->linestring.back()); - if (reverse_line) { - bg::reverse(way->linestring); - } - } - } - lastLinestring = way->linestring; + // If a reference is not on Way cache, skip this relation + if (!waycache.count(mit->ref)) { + noWay = true; + break; + } - ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); - geometry = ss.str(); - } - geometry.erase(0,11); + auto way = waycache.at(mit->ref); + + if (first) { + firstLinestringPoint = point_t( + boost::geometry::get<0>(way->linestring[0]), + boost::geometry::get<1>(way->linestring[0]) + ); + } + + // 1. When + // A) Relation is a LineString or MultiLineString but + // we want to save it as a MultiPolygon (this is the case for boundaries) + // B) the Relation is MultiPolygon but is composed of several LineStrings + if (using_linestring_tmp || (isMultiPolygon && + boost::geometry::num_points(way->linestring) > 0 && + boost::geometry::num_points(way->polygon) == 0) + ) { + + using_linestring_tmp = true; + std::stringstream ss; + std::string geometry; + + // If way's geometry is a polygon, save the outer + if (boost::geometry::num_points(way->polygon) > 0) { + ss << std::setprecision(12) << boost::geometry::wkt(way->polygon.outer()); + geometry = ss.str(); geometry.erase(geometry.size() - 1); - linestring_tmp += "(" + geometry + "),"; + // Way's geometry is a LineString } else { - // MultiPolygon or MultiLineString - - // When Relation is MultiLineString but way's geometry is Polygon - if (isMultiLineString && boost::geometry::num_points(way->linestring) == 0 && - boost::geometry::num_points(way->polygon) > 0 - ) { - // Convert way's Polygon to LineString - bg::assign_points(way->linestring, way->polygon.outer()); + if (first) { + std::cout << "Check if first linestring have to be reversed" << std::endl; + auto nextWay = waycache.at(std::next(mit)->ref); + + auto first_node_way1 = way->refs.front(); + auto first_node_way2 = nextWay->refs.front(); + auto last_node_way2 = nextWay->refs.back(); + + if (first_node_way1 == first_node_way2 || + first_node_way1 == last_node_way2) { + std::cout << "Yes! reverse first line" << std::endl; + bg::reverse(way->linestring); + firstLinestringPoint = point_t( + boost::geometry::get<0>(way->linestring.front()), + boost::geometry::get<1>(way->linestring.front()) + ); + } } - std::stringstream ss; - std::string geometry; + if (!first) { + // Reverse the line direction if it's necessary + if (lastLinestring.size() > 0 && way->linestring.size() > 0) { + bool reverse_line = bg::equals(lastLinestring.front(), way->linestring.front()) || + bg::equals(lastLinestring.front(), way->linestring.back()) || + bg::equals(lastLinestring.back(), way->linestring.front()) || + bg::equals(lastLinestring.back(), way->linestring.back()); + if (reverse_line) { + bg::reverse(way->linestring); + } + } - if (isMultiPolygon) { - ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); - geometry = ss.str(); - // Erase "POLYGON(" - geometry.erase(0,8); - } else { - ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); - geometry = ss.str(); - // Erase "LINESTRING(" - geometry.erase(0,11); - } - - // Erase ")" - geometry.erase(geometry.size() - 1); - - if (isMultiLineString) { - geometry = "(" + geometry + ")"; - } + // Check if object is closed - if (first && (mit->role != "inner")) { - first = false; - geometry_str += geometry + ","; - } else { - if (mit->role == "inner" && isMultiPolygon) { - innerGeoStr += geometry + ","; - } else { - geometry_str += geometry + ","; - geometry_str += innerGeoStr; - innerGeoStr = ""; + // FIXME: + // If one linestring reaches the beginning of the multilinestring + // take it as a polygon and start with a new one, to solve cases + // like this one https://www.openstreetmap.org/relation/16193116 + + if (bg::equals(way->linestring.back(), firstLinestringPoint)) { + close = true; } + } + lastLinestring = way->linestring; + ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); + geometry = ss.str(); } - } - } - if (!noWay && !noWayMembers) { - if (linestring_tmp.size() == 0) { - if (isMultiLineString) { - geometry_str.erase(geometry_str.size() - 1); - geometry_str = "MULTILINESTRING(" + geometry_str; - } else { - geometry_str = "MULTIPOLYGON((" + geometry_str; + // Erase "LINESTRING(" + geometry.erase(0,11); + geometry.erase(geometry.size() - 1); + // Get geometry coordinates as a string (lat lon, lat lon, ...), + if (close) { + std::cout << "***** [CLOSED OBJECT]" << std::endl; + std::cout << way->id << std::endl; } - if (innerGeoStr.size() > 0) { - geometry_str += innerGeoStr; - } - geometry_str.erase(geometry_str.size() - 1); - geometry_str += "))"; + linestring_tmp += "(" + geometry + "),"; + } else { - linestring_tmp.erase(linestring_tmp.size() - 1); - geometry_str = "MULTILINESTRING(" + linestring_tmp + ")"; - - boost::geometry::read_wkt(geometry_str, relation.multilinestring); - linestring_t mergedLineString; - for (auto& line : relation.multilinestring) { - bg::append(mergedLineString, line); + + // 2. Any other MultiPolygon or MultiLineString + + // When Relation is MultiLineString but way's geometry is a Polygon + if (!isMultiPolygon && boost::geometry::num_points(way->linestring) == 0 && + boost::geometry::num_points(way->polygon) > 0 + ) { + // Convert way's Polygon to LineString + bg::assign_points(way->linestring, way->polygon.outer()); } - // Create a polygon from the linestring - polygon_t polygon; - bg::append(polygon.outer(), mergedLineString); + std::stringstream ss; - ss << std::setprecision(12) << boost::geometry::wkt(polygon); - geometry_str = ss.str(); - // Erase "POLYGON" - geometry_str.erase(0, 8); - geometry_str = "MULTIPOLYGON((" + geometry_str + ")"; + std::string geometry; + if (isMultiPolygon) { + ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); + geometry = ss.str(); + // Erase "POLYGON(" + geometry.erase(0,8); + } else { + ss << std::setprecision(12) << boost::geometry::wkt(way->linestring); + geometry = ss.str(); + // Erase "LINESTRING(" + geometry.erase(0,11); + } + // Erase ")" + geometry.erase(geometry.size() - 1); + + + // Get geometry coordinates as a string (lat lon, lat lon, ...) + if (!isMultiPolygon) { + geometry = "(" + geometry + ")"; + } + + // Add way's geometry to the final result + // FIXME CHECK + if (first && (mit->role != "inner")) { + geometry_str += geometry + ","; + } else { + if (mit->role == "inner" && isMultiPolygon) { + innerGeoStr += geometry + ","; + } else { + geometry_str += geometry + ","; + geometry_str += innerGeoStr; + innerGeoStr = ""; + } + } } + first = false; + } + } - if (isMultiPolygon) { - boost::geometry::read_wkt(geometry_str, relation.multipolygon); + // If the relation has way members and all ways were found in the ways cache + if (!noWay && !noWayMembers) { + + // FIXME CHECK + if (linestring_tmp.size() == 0) { + if (!isMultiPolygon) { + geometry_str.erase(geometry_str.size() - 1); + geometry_str = "MULTILINESTRING(" + geometry_str; } else { - boost::geometry::read_wkt(geometry_str, relation.multilinestring); + geometry_str = "MULTIPOLYGON((" + geometry_str; } + if (innerGeoStr.size() > 0) { + geometry_str += innerGeoStr; + } + geometry_str.erase(geometry_str.size() - 1); + geometry_str += "))"; + + // FIXME CHECK + } else { + + std::cout << "HERE!" << std::endl; + + // Create a MultiLineString + // TODO: do this for each part + linestring_tmp.erase(linestring_tmp.size() - 1); + geometry_str = "MULTILINESTRING(" + linestring_tmp + ")"; + boost::geometry::read_wkt(geometry_str, relation.multilinestring); + linestring_t mergedLineString; + for (auto& line : relation.multilinestring) { + bg::append(mergedLineString, line); + } + + // Create a Polygon from the MultiLineString + polygon_t polygon; + bg::append(polygon.outer(), mergedLineString); + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(polygon); + geometry_str = ss.str(); + + // Erase "POLYGON" + geometry_str.erase(0, 8); + + // Create final MultiPolygon + geometry_str = "MULTIPOLYGON((" + geometry_str + ")"; } + + // Save the final geometry string + if (isMultiPolygon) { + boost::geometry::read_wkt(geometry_str, relation.multipolygon); + } else { + boost::geometry::read_wkt(geometry_str, relation.multilinestring); + } + } } diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index 468c8a19a..83ef74e8f 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -239,14 +239,6 @@ class OsmRelation : public OsmObject { ); }; - /// Relation can be componed of open ways, resulting in a multilinestring - bool isMultiLineString(void) const - { - return (tags.count("type") && - (tags.at("type") != "multipolygon") - ); - }; - }; } // namespace osmobjects diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 127fadba3..eda7a4c6b 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -511,10 +511,10 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } } - for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { - OsmRelation *relation = rel_it->get(); - removedRelations.push_back(relation->id); - } + // for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + // OsmRelation *relation = rel_it->get(); + // removedRelations.push_back(relation->id); + // } } // Add indirectly modified ways to osmchanges @@ -541,20 +541,20 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } // Add indirectly modified relations to osmchanges - if (modifiedWaysIds.size() > 1) { - modifiedWaysIds.erase(modifiedWaysIds.size() - 1); - auto modifiedRelations = getRelationsByWaysRefs(modifiedWaysIds); - auto change = std::make_shared(none); - for (auto rel_it = modifiedRelations.begin(); rel_it != modifiedRelations.end(); ++rel_it) { - auto relation = std::make_shared(*rel_it->get()); - // If the relation is not marked as removed, mark it as modified - if (std::find(removedRelations.begin(), removedRelations.end(), relation->id) == removedRelations.end()) { - relation->action = osmobjects::modify; - change->relations.push_back(relation); - } - } - osmchanges->changes.push_back(change); - } + // if (modifiedWaysIds.size() > 1) { + // modifiedWaysIds.erase(modifiedWaysIds.size() - 1); + // auto modifiedRelations = getRelationsByWaysRefs(modifiedWaysIds); + // auto change = std::make_shared(none); + // for (auto rel_it = modifiedRelations.begin(); rel_it != modifiedRelations.end(); ++rel_it) { + // auto relation = std::make_shared(*rel_it->get()); + // // If the relation is not marked as removed, mark it as modified + // if (std::find(removedRelations.begin(), removedRelations.end(), relation->id) == removedRelations.end()) { + // relation->action = osmobjects::modify; + // change->relations.push_back(relation); + // } + // } + // osmchanges->changes.push_back(change); + // } // Fill nodecache with referenced nodes if (referencedNodeIds.size() > 1) { @@ -599,44 +599,44 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } // Relations - std::string relsForWayCacheIds; - bool debug = false; - for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { - OsmChange *change = it->get(); - for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { - OsmRelation *relation = rel_it->get(); - if (relation->isMultiPolygon() || relation->isMultiLineString()) { - bool getWaysForRelation = false; - for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { - if (osmchanges->waycache.count(mit->ref)) { - getWaysForRelation = true; - break; - } - } - if (getWaysForRelation) { - for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { - if (!osmchanges->waycache.count(mit->ref)) { - relsForWayCacheIds += std::to_string(mit->ref) + ","; - } - } - } - } - } - } - // Get all missing ways geometries for relations - if (relsForWayCacheIds != "") { - relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); - getWaysByIds(relsForWayCacheIds, osmchanges->waycache); - } + // std::string relsForWayCacheIds; + // bool debug = false; + // for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { + // OsmChange *change = it->get(); + // for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + // OsmRelation *relation = rel_it->get(); + // if (relation->isMultiPolygon()) { + // bool getWaysForRelation = false; + // for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + // if (osmchanges->waycache.count(mit->ref)) { + // getWaysForRelation = true; + // break; + // } + // } + // if (getWaysForRelation) { + // for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + // if (!osmchanges->waycache.count(mit->ref)) { + // relsForWayCacheIds += std::to_string(mit->ref) + ","; + // } + // } + // } + // } + // } + // } + // // Get all missing ways geometries for relations + // if (relsForWayCacheIds != "") { + // relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); + // getWaysByIds(relsForWayCacheIds, osmchanges->waycache); + // } // Build relation geometries - for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { - OsmChange *change = it->get(); - for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { - OsmRelation *relation = rel_it->get(); - osmchanges->buildRelationGeometry(*relation); - } - } + // for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { + // OsmChange *change = it->get(); + // for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + // OsmRelation *relation = rel_it->get(); + // osmchanges->buildRelationGeometry(*relation); + // } + // } } void diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 0e64f45bf..711faf9e5 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -495,7 +495,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) auto removed_nodes = std::make_shared>(); auto removed_ways = std::make_shared>(); - auto removed_relations = std::make_shared>(); + // auto removed_relations = std::make_shared>(); auto validation_removals = std::make_shared>(); // Raw data and validation @@ -541,23 +541,23 @@ threadOsmChange(OsmChangeTask osmChangeTask) } } - // Relations - for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { - osmobjects::OsmRelation *relation = rit->get(); + // // Relations + // for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + // osmobjects::OsmRelation *relation = rit->get(); - if (relation->action != osmobjects::remove && !relation->priority) { - continue; - } - // Remove deleted relations from validation table - if (!config->disable_validation && relation->action == osmobjects::remove) { - removed_relations->push_back(relation->id); - } + // if (relation->action != osmobjects::remove && !relation->priority) { + // continue; + // } + // // Remove deleted relations from validation table + // if (!config->disable_validation && relation->action == osmobjects::remove) { + // removed_relations->push_back(relation->id); + // } - // Update relations, ignore new ones outside priority area - if (!config->disable_raw) { - task.query += queryraw->applyChange(*relation); - } - } + // // Update relations, ignore new ones outside priority area + // if (!config->disable_raw) { + // task.query += queryraw->applyChange(*relation); + // } + // } } } @@ -580,7 +580,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) task.query += queryvalidate->updateValidation(validation_removals); task.query += queryvalidate->updateValidation(removed_nodes); task.query += queryvalidate->updateValidation(removed_ways); - task.query += queryvalidate->updateValidation(removed_relations); + // task.query += queryvalidate->updateValidation(removed_relations); } diff --git a/src/testsuite/libunderpass.all/areafilter-test.cc b/src/testsuite/libunderpass.all/areafilter-test.cc index 1b5de7a79..d0d2e03d9 100644 --- a/src/testsuite/libunderpass.all/areafilter-test.cc +++ b/src/testsuite/libunderpass.all/areafilter-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -86,11 +86,6 @@ getPriority(TestOsmChange &osmchange, bool debug = false) { return result; } -void -clearChanges(TestOsmChange &osmchange) { - osmchange.changes.clear(); -} - int main(int argc, char *argv[]) { @@ -167,57 +162,50 @@ main(int argc, char *argv[]) osmchange.readChanges(osmchangeFile); osmchange.buildGeometriesFromNodeCache(); - // osmchange.areaFilter(polyEmpty); - // if (getPriority(osmchange) && countFeatures(osmchange) == 62) { - // runtest.pass("OsmChange areaFilter - true (Empty poly)"); - // } else { - // runtest.fail("OsmChange areaFilter - true (Empty poly)"); - // return 1; - // } - - // // OsmChange - Whole world - // osmchange.areaFilter(polyWholeWorld); - // if (getPriority(osmchange) && countFeatures(osmchange) == 62) { - // runtest.pass("OsmChange areaFilter - true (whole world)"); - // } else { - // runtest.fail("OsmChange areaFilter - true (whole world)"); - // return 1; - // } + osmchange.areaFilter(polyEmpty); + if (getPriority(osmchange) && countFeatures(osmchange) == 54) { + runtest.pass("OsmChange areaFilter - 54 (empty poly)"); + } else { + runtest.fail("OsmChange areaFilter - 54 (empty poly)"); + return 1; + } + + // OsmChange - Whole world + osmchange.areaFilter(polyWholeWorld); + if (getPriority(osmchange) && countFeatures(osmchange) == 54) { + runtest.pass("OsmChange areaFilter - 54 (whole world)"); + } else { + runtest.fail("OsmChange areaFilter - 54 (Whole world)"); + return 1; + } // OsmChange - Small area in North Africa // Outside priority area, count should be 0 - // clearChanges(osmchange); - // osmchange.readChanges(osmchangeFile); osmchange.areaFilter(polySmallArea); if (countFeatures(osmchange) == 0) { - runtest.pass("OsmChange areaFilter - true (0)"); + runtest.pass("OsmChange areaFilter - 0 (Small area)"); } else { - runtest.fail("OsmChange areaFilter - true (0)"); + runtest.fail("OsmChange areaFilter - 0 (small area)"); return 1; } // OsmChange - Small area in Bangladesh // 28 nodes / 5 ways / 1 relation inside priority area, count should be 34 - clearChanges(osmchange); - osmchange.readChanges(osmchangeFile); - osmchange.buildGeometriesFromNodeCache(); osmchange.areaFilter(polyHalf); if (countFeatures(osmchange) == 34) { - runtest.pass("OsmChange areaFilter - true (34)"); + runtest.pass("OsmChange areaFilter - 34 (small area)"); } else { - runtest.fail("OsmChange areaFilter - true (34)"); + runtest.fail("OsmChange areaFilter - 34 (sSmall area)"); return 1; } // OsmChange - Smaller area in Bangladesh // 12 nodes / 3 ways / 1 relation inside priority area, count should be 16 - clearChanges(osmchange); - osmchange.readChanges(osmchangeFile); osmchange.areaFilter(polyHalfSmall); if (countFeatures(osmchange) == 16) { - runtest.pass("OsmChange areaFilter - true (smaller area inside)"); + runtest.pass("OsmChange areaFilter - 16 (smaller area)"); } else { - runtest.fail("OsmChange areaFilter - true (smaller area inside)"); + runtest.fail("OsmChange areaFilter - 16 (smaller area)"); return 1; } diff --git a/src/testsuite/libunderpass.all/raw-test.cc b/src/testsuite/libunderpass.all/raw-test.cc index ba2032156..229f4bbe1 100644 --- a/src/testsuite/libunderpass.all/raw-test.cc +++ b/src/testsuite/libunderpass.all/raw-test.cc @@ -113,7 +113,9 @@ const std::vector expectedGeometries = { "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))", "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.7260807753 4.62037032501,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))", "MULTILINESTRING((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836),(21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))", - "MULTIPOLYGON(((-59.8355946 -36.7448782,-59.8353203 -36.7452085,-59.8368532 -36.7480926,-59.8370667 -36.7483355,-59.8408639 -36.7515154,-59.8482593 -36.745907,-59.8486299 -36.7452032,-59.848532 -36.7453269,-59.8483958 -36.7456167,-59.8482593 -36.745907,-59.8486299 -36.7452032,-59.8486213 -36.7451521,-59.8481455 -36.7432981,-59.8478834 -36.7425679,-59.8478326 -36.7423963,-59.8478326 -36.7423963,-59.8477902 -36.7422529,-59.8477764 -36.7422007,-59.8477733 -36.74215,-59.8477773 -36.742095,-59.8477972 -36.7420329,-59.8478247 -36.7419783,-59.8478609 -36.7419236,-59.8479126 -36.7418808,-59.84795 -36.7418607,-59.8480462 -36.7418147,-59.8480692 -36.7417819,-59.8480709 -36.7417548,-59.8480673 -36.7416893,-59.8479835 -36.741163,-59.8479787 -36.7411211,-59.8479562 -36.7410819,-59.847923 -36.7410414,-59.8478722 -36.7410094,-59.8475273 -36.7408127,-59.8474352 -36.7407496,-59.8473715 -36.7406912,-59.8473254 -36.7406329,-59.8472829 -36.7405588,-59.8472494 -36.7404962,-59.8472589 -36.7404251,-59.8472978 -36.7403582,-59.8473424 -36.74032,-59.8473864 -36.7402921,-59.8474375 -36.740274,-59.8474922 -36.740256,-59.8475468 -36.7402465,-59.8476241 -36.7402304,-59.8476784 -36.7402171,-59.8477563 -36.7401919,-59.8478265 -36.7401545,-59.8479128 -36.7400779,-59.8479937 -36.7399885,-59.8480801 -36.7398796,-59.8481504 -36.7397665,-59.848411 -36.7391216,-59.8484023 -36.739028,-59.8483557 -36.7389343,-59.8482783 -36.7388702,-59.84822 -36.7388242,-59.8481308 -36.7388079,-59.8480003 -36.7387583,-59.8478575 -36.7386841,-59.8477933 -36.738618,-59.8477365 -36.7385158,-59.8477106 -36.7384235,-59.8477053 -36.7382963,-59.8477134 -36.7381998,-59.8477433 -36.7381126,-59.8478321 -36.738022,-59.8479105 -36.7379644,-59.8480011 -36.7379216,-59.8482127 -36.7378122,-59.8482877 -36.7377698,-59.8483566 -36.7377126,-59.848393 -36.737632,-59.8484294 -36.7375366,-59.8485761 -36.7372026,-59.848605 -36.7370773,-59.8486246 -36.7369372,-59.8486196 -36.7368366,-59.8485807 -36.7367015,-59.848529 -36.7365836,-59.8484717 -36.7364835,-59.8483887 -36.7363497,-59.8477548 -36.7356502,-59.8477339 -36.7356248,-59.8477339 -36.7356248,-59.8475634 -36.7357007,-59.8474292 -36.7357691,-59.8473073 -36.7358571,-59.8469617 -36.7361243,-59.8447338 -36.737825,-59.8424572 -36.7395354,-59.8423067 -36.7396527,-59.8386641 -36.7424968,-59.838225 -36.7428388,-59.8355946 -36.7448782)))" + "MULTIPOLYGON(((-59.8355946 -36.7448782,-59.8353203 -36.7452085,-59.8368532 -36.7480926,-59.8370667 -36.7483355,-59.8408639 -36.7515154,-59.8482593 -36.745907,-59.8486299 -36.7452032,-59.848532 -36.7453269,-59.8483958 -36.7456167,-59.8482593 -36.745907,-59.8486299 -36.7452032,-59.8486213 -36.7451521,-59.8481455 -36.7432981,-59.8478834 -36.7425679,-59.8478326 -36.7423963,-59.8478326 -36.7423963,-59.8477902 -36.7422529,-59.8477764 -36.7422007,-59.8477733 -36.74215,-59.8477773 -36.742095,-59.8477972 -36.7420329,-59.8478247 -36.7419783,-59.8478609 -36.7419236,-59.8479126 -36.7418808,-59.84795 -36.7418607,-59.8480462 -36.7418147,-59.8480692 -36.7417819,-59.8480709 -36.7417548,-59.8480673 -36.7416893,-59.8479835 -36.741163,-59.8479787 -36.7411211,-59.8479562 -36.7410819,-59.847923 -36.7410414,-59.8478722 -36.7410094,-59.8475273 -36.7408127,-59.8474352 -36.7407496,-59.8473715 -36.7406912,-59.8473254 -36.7406329,-59.8472829 -36.7405588,-59.8472494 -36.7404962,-59.8472589 -36.7404251,-59.8472978 -36.7403582,-59.8473424 -36.74032,-59.8473864 -36.7402921,-59.8474375 -36.740274,-59.8474922 -36.740256,-59.8475468 -36.7402465,-59.8476241 -36.7402304,-59.8476784 -36.7402171,-59.8477563 -36.7401919,-59.8478265 -36.7401545,-59.8479128 -36.7400779,-59.8479937 -36.7399885,-59.8480801 -36.7398796,-59.8481504 -36.7397665,-59.848411 -36.7391216,-59.8484023 -36.739028,-59.8483557 -36.7389343,-59.8482783 -36.7388702,-59.84822 -36.7388242,-59.8481308 -36.7388079,-59.8480003 -36.7387583,-59.8478575 -36.7386841,-59.8477933 -36.738618,-59.8477365 -36.7385158,-59.8477106 -36.7384235,-59.8477053 -36.7382963,-59.8477134 -36.7381998,-59.8477433 -36.7381126,-59.8478321 -36.738022,-59.8479105 -36.7379644,-59.8480011 -36.7379216,-59.8482127 -36.7378122,-59.8482877 -36.7377698,-59.8483566 -36.7377126,-59.848393 -36.737632,-59.8484294 -36.7375366,-59.8485761 -36.7372026,-59.848605 -36.7370773,-59.8486246 -36.7369372,-59.8486196 -36.7368366,-59.8485807 -36.7367015,-59.848529 -36.7365836,-59.8484717 -36.7364835,-59.8483887 -36.7363497,-59.8477548 -36.7356502,-59.8477339 -36.7356248,-59.8477339 -36.7356248,-59.8475634 -36.7357007,-59.8474292 -36.7357691,-59.8473073 -36.7358571,-59.8469617 -36.7361243,-59.8447338 -36.737825,-59.8424572 -36.7395354,-59.8423067 -36.7396527,-59.8386641 -36.7424968,-59.838225 -36.7428388,-59.8355946 -36.7448782)))", + "MULTIPOLYGON(((-69.0344971 -33.6875005,-69.0354574 -33.6880228,-69.0356076 -33.6879112,-69.0360421 -33.6881656,-69.0362352 -33.6883218,-69.0369111 -33.6886075,-69.0375173 -33.6871078,-69.0367931 -33.686581,-69.0366161 -33.6866525,-69.0361923 -33.6865766,-69.0364122 -33.6860945,-69.0368253 -33.68634,-69.0370399 -33.6864739,-69.037512 -33.6861168,-69.0374959 -33.6859115,-69.0376568 -33.6857687,-69.037866 -33.6856838,-69.037351 -33.6853401,-69.0371311 -33.6842242,-69.0365088 -33.68376,-69.0362889 -33.6832065,-69.036144 -33.6823673,-69.0358865 -33.6818227,-69.0358973 -33.6817468,-69.03557 -33.6815816,-69.0359187 -33.6810459,-69.0351462 -33.6804031,-69.0349263 -33.6798541,-69.0345454 -33.67968,-69.0342611 -33.6794791,-69.0337515 -33.6794836,-69.0332151 -33.6793095,-69.0331185 -33.6788586,-69.0329737 -33.6786354,-69.0327162 -33.6783541,-69.0325767 -33.6782024,-69.0321851 -33.6780238,-69.0315521 -33.6776756,-69.0312892 -33.6774881,-69.0311068 -33.6773453,-69.0308118 -33.6771131,-69.0305275 -33.6769747,-69.0303397 -33.6769033,-69.0300447 -33.676948,-69.0298086 -33.6769256,-69.0288216 -33.6784256,-69.0287519 -33.6783742,-69.0296692 -33.6768676,-69.0292937 -33.6765819,-69.0289557 -33.6765596,-69.0286982 -33.6766355,-69.0284568 -33.6765685,-69.0282691 -33.6768363,-69.0279794 -33.6769122,-69.0277594 -33.6767917,-69.0276629 -33.6770015,-69.0274376 -33.6768988,-69.0273035 -33.6768899,-69.0270782 -33.6770328,-69.0268743 -33.6769301,-69.0265229 -33.6775261,-69.0258417 -33.6786086,-69.0233284 -33.6774703,-69.0232372 -33.6775238,-69.0232211 -33.6781176,-69.0231353 -33.6783408,-69.0233445 -33.6788229,-69.0233767 -33.6791979,-69.0232158 -33.6794925,-69.0233231 -33.6796264,-69.022733 -33.6803049,-69.0227705 -33.6806933,-69.0227598 -33.6811084,-69.0223092 -33.6817245,-69.0238702 -33.6824744,-69.0240419 -33.6822735,-69.0255761 -33.6830234,-69.0254581 -33.6832243,-69.0291756 -33.6849295,-69.0293527 -33.6846617,-69.0311283 -33.6854607,-69.0308976 -33.6858222,-69.0337139 -33.6871256,-69.0343523 -33.6862061,-69.0349907 -33.6865409,-69.0343738 -33.6874604,-69.0344971 -33.6875005),(-69.0305328 -33.683019,-69.0311444 -33.6821173,-69.0274215 -33.6803451,-69.0263647 -33.6819119,-69.0305328 -33.683019)))", + "MULTIPOLYGON(((-70.3434661 -38.5210725,-70.3434387 -38.5210955,-70.3434261 -38.5211299,-70.3434276 -38.5211716,-70.3434429 -38.5212067,-70.3434724 -38.5212373,-70.3435173 -38.5212595,-70.3435951 -38.5212788,-70.3436942 -38.5212955,-70.3438252 -38.52131479999999,-70.3439597 -38.521337,-70.3441048 -38.521359100000005,-70.3442463 -38.52138410000001,-70.3443808 -38.5214145,-70.3445082 -38.5214533,-70.3446177 -38.521495599999994,-70.3447356 -38.52154330000001,-70.3448535 -38.52158469999999,-70.3449795 -38.5216101,-70.3451015 -38.521622900000004,-70.3452113 -38.52164189999999,-70.3453088 -38.5216674,-70.3454192 -38.521701,-70.3455168 -38.5217296,-70.3456184 -38.5217582,-70.3457445 -38.5217805,-70.3458786 -38.52179,-70.3460209 -38.52179,-70.3471381 -38.5217576,-70.3472051 -38.52175339999999,-70.3472561 -38.52174300000001,-70.3473044 -38.52172410000001,-70.3473446 -38.521701,-70.3473795 -38.52166949999999,-70.3474116 -38.5216422,-70.3474492 -38.521612800000014,-70.3474921 -38.5215835,-70.3475323 -38.5215541,-70.3476101 -38.52149739999999,-70.3475739 -38.52162409999999,-70.3475456 -38.521693299999995,-70.3475137 -38.52176259999999,-70.3474677 -38.5218235,-70.3474253 -38.5218674,-70.3473686 -38.52190379999999,-70.3473085 -38.5219342,-70.3472413 -38.521954,-70.347174 -38.5219675,-70.3472817 -38.5219659,-70.3474005 -38.5219536,-70.3475173 -38.52192869999999,-70.3476447 -38.5218955,-70.3477721 -38.52184559999999,-70.3479455 -38.52176259999999,-70.3479066 -38.52183180000001,-70.3478641 -38.5218789,-70.3478075 -38.52192869999999,-70.3477355 -38.5219766,-70.3479018 -38.5219262,-70.3480467 -38.5218675,-70.3481915 -38.521791900000004,-70.3483042 -38.52172060000001,-70.3484135 -38.521645,-70.3486227 -38.5215149,-70.3488534 -38.52138899999999,-70.3491002 -38.5212799,-70.3493684 -38.5211918,-70.3496098 -38.5211162,-70.3498834 -38.5210491,-70.3501301 -38.5210029,-70.3503447 -38.5209693,-70.3505593 -38.520944100000015,-70.3507792 -38.5209204,-70.3507792 -38.5209204,-70.3505605 -38.52082279999999,-70.350236 -38.5206906,-70.3498659 -38.5205962,-70.3495735 -38.5205458,-70.3492637 -38.5204975,-70.3491282 -38.52049329999999,-70.3490075 -38.520505899999996,-70.3488064 -38.5205469,-70.3488064 -38.5205469,-70.3485167 -38.5206098,-70.3478837 -38.5207179,-70.3473875 -38.5207808,-70.3468376 -38.5208333,-70.3461456 -38.5208816,-70.3455448 -38.520934,-70.3449628 -38.5209949,-70.3443834 -38.52103060000001,-70.343796 -38.52105360000001,-70.3434661 -38.5210725)),((-70.3393449 -38.5210096,-70.3397998 -38.5209924,-70.3400091 -38.5209944,-70.3401816 -38.521013700000005,-70.3403726 -38.5210619,-70.340539 -38.52112459999999,-70.3406807 -38.5211969,-70.3408225 -38.5212789,-70.3409704 -38.5213898,-70.3410954 -38.521514500000016,-70.341237 -38.5216474,-70.3414069 -38.5217803,-70.3415626 -38.52188550000001,-70.3417466 -38.52196860000001,-70.3419731 -38.5220406,-70.3421855 -38.522096,-70.3423908 -38.5221458,-70.3426173 -38.5222012,-70.3428155 -38.522262099999985,-70.3430137 -38.52233410000001,-70.3432119 -38.5224338,-70.3433534 -38.5225058,-70.343495 -38.5225889,-70.3436649 -38.522660899999984,-70.3438772 -38.5226885,-70.3440683 -38.522682999999994,-70.3442524 -38.522660899999984,-70.3449444 -38.522532899999995,-70.3450571 -38.5224952,-70.3451644 -38.522428000000005,-70.3452341 -38.5223483,-70.3452663 -38.5222769,-70.3452717 -38.52220139999999,-70.3452448 -38.5221342,-70.3451966 -38.5220797,-70.3451 -38.5220335,-70.3449444 -38.52198729999999,-70.3447513 -38.52195369999999,-70.3445624 -38.52194130000001,-70.343915 -38.52190679999999,-70.3437005 -38.5218859,-70.3434859 -38.5218565,-70.3432337 -38.5217935,-70.3429923 -38.521717999999986,-70.3427402 -38.5216215,-70.3425364 -38.5215333,-70.3423486 -38.5214368,-70.3421877 -38.521336100000006,-70.3420589 -38.5212353,-70.341948 -38.5211271,-70.341948 -38.5211271,-70.3415966 -38.5210893,-70.3412157 -38.521032700000006,-70.3409931 -38.5209875,-70.3408737 -38.5209414,-70.3406967 -38.5208417,-70.3405653 -38.5207452,-70.3404205 -38.5206245,-70.3402944 -38.520521699999996,-70.3401375 -38.52040730000002,-70.3399175 -38.520231,-70.3397392 -38.5200925,-70.3396024 -38.5200075,-70.3394187 -38.519918299999986,-70.3392309 -38.519868,-70.3388715 -38.5197882,-70.3384933 -38.5197012,-70.3380534 -38.5196015,-70.3377664 -38.519549000000005,-70.3372943 -38.519500699999995,-70.3368531 -38.5194703,-70.3365098 -38.5194577,-70.3361759 -38.5194399,-70.3359184 -38.5194567,-70.3359184 -38.5194567,-70.3362585 -38.519638999999984,-70.3365107 -38.5197779,-70.3367348 -38.51993860000001,-70.3369777 -38.520114,-70.3372112 -38.5202456,-70.3376501 -38.5204356,-70.3381078 -38.520632899999995,-70.3384721 -38.5208009,-70.3387149 -38.520874,-70.3390231 -38.52093250000001,-70.3393449 -38.5210096)))" }; std::string @@ -151,84 +153,92 @@ main(int argc, char *argv[]) auto queryraw = std::make_shared(db); std::map> waycache; - processFile("raw-case-1.osc", db); - processFile("raw-case-2.osc", db); + // processFile("raw-case-1.osc", db); + // processFile("raw-case-2.osc", db); - std::string waysIds = "101874,101875"; - queryraw->getWaysByIds(waysIds, waycache); + // std::string waysIds = "101874,101875"; + // queryraw->getWaysByIds(waysIds, waycache); - // 4 created Nodes, 1 created Way (same changeset) - if ( getWKT(waycache.at(101874)->polygon).compare(expectedGeometries[0]) == 0) { - runtest.pass("4 created Nodes, 1 created Ways (same changeset)"); - } else { - runtest.fail("4 created Nodes, 1 created Ways (same changeset)"); - return 1; - } + // // 4 created Nodes, 1 created Way (same changeset) + // if ( getWKT(waycache.at(101874)->polygon).compare(expectedGeometries[0]) == 0) { + // runtest.pass("4 created Nodes, 1 created Ways (same changeset)"); + // } else { + // runtest.fail("4 created Nodes, 1 created Ways (same changeset)"); + // return 1; + // } - // 1 created Way, 4 existing Nodes (different changeset) - if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[1]) == 0) { - runtest.pass("1 created Way, 4 existing Nodes (different changesets)"); - } else { - runtest.fail("1 created Way, 4 existing Nodes (different changesets)"); - return 1; - } + // // 1 created Way, 4 existing Nodes (different changeset) + // if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[1]) == 0) { + // runtest.pass("1 created Way, 4 existing Nodes (different changesets)"); + // } else { + // runtest.fail("1 created Way, 4 existing Nodes (different changesets)"); + // return 1; + // } - // 1 modified node, indirectly modify other existing ways - processFile("raw-case-3.osc", db); - waycache.erase(101875); - queryraw->getWaysByIds(waysIds, waycache); - if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[2]) == 0) { - runtest.pass("1 modified Node, indirectly modify other existing Ways (different changesets)"); - } else { - runtest.fail("1 modified Node, indirectly modify other existing Ways (different changesets)"); - return 1; - } + // // 1 modified node, indirectly modify other existing ways + // processFile("raw-case-3.osc", db); + // waycache.erase(101875); + // queryraw->getWaysByIds(waysIds, waycache); + // if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries[2]) == 0) { + // runtest.pass("1 modified Node, indirectly modify other existing Ways (different changesets)"); + // } else { + // runtest.fail("1 modified Node, indirectly modify other existing Ways (different changesets)"); + // return 1; + // } - // 1 created Relation referencing 1 created Way and 1 existing Way - processFile("raw-case-4.osc", db); - if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[3]) == 0) { - runtest.pass("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); - } else { - runtest.fail("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); - return 1; - } + // // 1 created Relation referencing 1 created Way and 1 existing Way + // processFile("raw-case-4.osc", db); + // if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[3]) == 0) { + // runtest.pass("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); + // } else { + // runtest.fail("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); + // return 1; + // } - // 1 modified Node, indirectly modify other existing Ways and 1 Relation - processFile("raw-case-5.osc", db); - if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[4]) == 0) { - runtest.pass("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); - } else { - runtest.fail("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); - return 1; - } + // // 1 modified Node, indirectly modify other existing Ways and 1 Relation + // processFile("raw-case-5.osc", db); + // if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries[4]) == 0) { + // runtest.pass("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); + // } else { + // runtest.fail("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); + // return 1; + // } - // 4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring - processFile("raw-case-6.osc", db); - if ( getWKTFromDB("relations", 211776, db).compare(expectedGeometries[5]) == 0) { - runtest.pass("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); - } else { - runtest.fail("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); - return 1; - } + // // 4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring + // processFile("raw-case-6.osc", db); + // if ( getWKTFromDB("relations", 211776, db).compare(expectedGeometries[5]) == 0) { + // runtest.pass("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); + // } else { + // runtest.fail("4 created Nodes, 2 created Ways, 1 created Relation with type=multilinestring (same changeset)"); + // return 1; + // } - // Complex, 1 polygon relation made of multiple ways - processFile("raw-case-7.osc", db); - if ( getWKTFromDB("relations", 17331328, db).compare(expectedGeometries[6]) == 0) { - runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset)"); - } else { - runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset)"); - return 1; - } + // // Complex, 1 polygon relation made of multiple ways + // processFile("raw-case-7.osc", db); + // if ( getWKTFromDB("relations", 17331328, db).compare(expectedGeometries[6]) == 0) { + // runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset)"); + // } else { + // runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset)"); + // return 1; + // } // - processFile("raw-case-8.osc", db); - // if ( getWKTFromDB("relations", 17331328, db).compare(expectedGeometries[6]) == 0) { + // processFile("raw-case-8.osc", db); + // if ( getWKTFromDB("relations", 16191459, db).compare(expectedGeometries[7]) == 0) { // runtest.pass("Complex, 1 polygon relation made of multiple ways (same changeset)"); // } else { // runtest.fail("Complex, 1 polygon relation made of multiple ways (same changeset)"); // return 1; // } + processFile("raw-case-9.osc", db); + if ( getWKTFromDB("relations", 16193116, db).compare(expectedGeometries[8]) == 0) { + runtest.pass("Complex, 2 polygon relation made of multiple ways (same changeset)"); + } else { + runtest.fail("Complex, 2 polygon relation made of multiple ways (same changeset)"); + return 1; + } + } diff --git a/src/testsuite/testdata/raw/raw-case-7.osc b/src/testsuite/testdata/raw/raw-case-7.osc index 261b8228b..093fa6906 100644 --- a/src/testsuite/testdata/raw/raw-case-7.osc +++ b/src/testsuite/testdata/raw/raw-case-7.osc @@ -200,19 +200,13 @@ - - - - - - + + + + + + - - - - - - diff --git a/src/testsuite/testdata/raw/raw-case-8.osc b/src/testsuite/testdata/raw/raw-case-8.osc new file mode 100644 index 000000000..aa2554b04 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-8.osc @@ -0,0 +1,198 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/testsuite/testdata/raw/raw-case-9.osc b/src/testsuite/testdata/raw/raw-case-9.osc new file mode 100644 index 000000000..3c833f611 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-9.osc @@ -0,0 +1,380 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 2e6d9594e7cfd4501d4b7caf4c734d9a4346a615 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 25 Mar 2024 10:25:32 -0300 Subject: [PATCH 5/9] Fixes for main replicator, temporarily disable relations processing --- src/bootstrap/bootstrap.cc | 2 +- src/underpass.cc | 78 +++++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 7358ed5fc..d4c2d6ca2 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -93,7 +93,7 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) { processWays(); processNodes(); - processRelations(); + // processRelations(); } diff --git a/src/underpass.cc b/src/underpass.cc index d5082ddf7..f9d2bb50a 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -181,7 +181,7 @@ main(int argc, char *argv[]) config.concurrency = std::thread::hardware_concurrency(); } - if (vm.count("timestamp") || vm.count("url")) { + if (vm.count("timestamp") || vm.count("url") || vm.count("changeseturl")) { // Planet server if (vm.count("planet")) { @@ -198,6 +198,38 @@ main(int argc, char *argv[]) config.planet_server = config.planet_servers[0].domain; } + // Default data directory on the server + if (vm.count("datadir")) { + datadir = vm["datadir"].as(); + } + const char *tmp = std::getenv("DATADIR"); + if (tmp != 0) { + datadir = tmp; + } + config.datadir = datadir; + + // Frequency: minutely, hourly, daily + if (vm.count("frequency")) { + const auto strfreq = vm["frequency"].as(); + if (strfreq[0] == 'm') { + config.frequency = replication::minutely; + } else if (strfreq[0] == 'h') { + config.frequency = replication::hourly; + } else if (strfreq[0] == 'd') { + config.frequency = replication::daily; + } else { + log_debug("Invalid frequency!"); + exit(-1); + } + } + + planetreplicator::PlanetReplicator replicator; + replicator.connectServer("https://" + config.planet_server); + + + std::string fullurl = "https://" + config.planet_server + "/replication/" + StateFile::freq_to_string(config.frequency); + std::vector parts; + // Priority boundary multipolygon_t poly; if (vm.count("boundary")) { @@ -224,38 +256,11 @@ main(int argc, char *argv[]) } // Replication - planetreplicator::PlanetReplicator replicator; - std::shared_ptr> data; if (vm.count("url") && vm.count("timestamp")) { log_debug("ERROR: 'url' takes precedence over 'timestamp' arguments are mutually exclusive!"); exit(-1); } - // Default data directory on the server - if (vm.count("datadir")) { - datadir = vm["datadir"].as(); - } - const char *tmp = std::getenv("DATADIR"); - if (tmp != 0) { - datadir = tmp; - } - config.datadir = datadir; - - // Frequency: minutely, hourly, daily - if (vm.count("frequency")) { - const auto strfreq = vm["frequency"].as(); - if (strfreq[0] == 'm') { - config.frequency = replication::minutely; - } else if (strfreq[0] == 'h') { - config.frequency = replication::hourly; - } else if (strfreq[0] == 'd') { - config.frequency = replication::daily; - } else { - log_debug("Invalid frequency!"); - exit(-1); - } - } - auto osmchange = std::make_shared(); // Specify a timestamp used by other options if (vm.count("timestamp")) { @@ -275,12 +280,10 @@ main(int argc, char *argv[]) exit(-1); } } else if (vm.count("url")) { - replicator.connectServer("https://" + config.planet_server); std::string fullurl = "https://" + config.planet_server + "/replication/" + StateFile::freq_to_string(config.frequency); std::vector parts; boost::split(parts, vm["url"].as(), boost::is_any_of("/")); fullurl += "/" + vm["url"].as() + ".state.txt"; - osmchange->parse(fullurl); osmchange->destdir_base = config.destdir_base; auto data = replicator.downloadFile(*osmchange).data; @@ -306,19 +309,24 @@ main(int argc, char *argv[]) // Changesets std::thread changesetThread; - if (vm.count("changeseturl") || vm.count("timestamp")) { + if (vm.count("changeseturl")) { + auto changeset = std::make_shared(); + auto changeseturl = vm["changeseturl"].as(); config.frequency = replication::changeset; - auto changeset = replicator.findRemotePath(config, config.start_time); + std::string fullurl = "https://" + config.planet_server + + "/replication/" + StateFile::freq_to_string(config.frequency) + + "/" + changeseturl + ".osm.gz"; + changeset->parse(fullurl); changeset->destdir_base = config.destdir_base; std::vector parts; - boost::split(parts, vm["changeseturl"].as(), boost::is_any_of("/")); + boost::split(parts, changeseturl, boost::is_any_of("/")); changeset->updatePath(stoi(parts[0]),stoi(parts[1]),stoi(parts[2])); if (!config.silent) { changeset->dump(); } if (!vm.count("osmchanges")) { - changesetThread = std::thread(replicatorthreads::startMonitorChangesets, std::ref(changeset), - std::ref(*oscboundary), std::ref(config)); + changesetThread = std::thread(replicatorthreads::startMonitorChangesets, + std::ref(changeset), std::ref(*oscboundary), std::ref(config)); } } From a6afbd969f08aa6949bc4f1f4cdad63cb2931bc1 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 25 Mar 2024 12:11:03 -0300 Subject: [PATCH 6/9] Fixes for changeset processing --- src/osm/changeset.cc | 10 +--------- src/osm/osmchange.cc | 1 + src/replicator/planetreplicator.cc | 4 ---- src/underpass.cc | 6 +----- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/src/osm/changeset.cc b/src/osm/changeset.cc index 8a80738de..8cce25684 100644 --- a/src/osm/changeset.cc +++ b/src/osm/changeset.cc @@ -364,17 +364,9 @@ ChangeSetFile::on_start_element(const Glib::ustring &name, bool hashit = false; bool comhit = false; bool cbyhit = false; - double min_lat = 0.0; - double min_lon = 0.0; - double max_lat = 0.0; - double max_lon = 0.0; for (const auto &attr_pair: attributes) { - // std::wcout << "\tPAIR: " << attr_pair.name << " = " << std::endl; - // attr_pair.value << std::endl; - if (attr_pair.name == "k" && attr_pair.value == "max_lat") { - max_lat = std::stod(attr_pair.value); - } else if (attr_pair.name == "k" && attr_pair.value == "hashtags") { + if (attr_pair.name == "k" && attr_pair.value == "hashtags") { hashit = true; } else if (attr_pair.name == "k" && attr_pair.value == "comment") { comhit = true; diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index 76c459b5c..d9f87ebe7 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -508,6 +508,7 @@ void OsmChangeFile::on_start_element(const Glib::ustring &name, const AttributeList &attributes) { + // If a change is in progress, apply to to that instance std::shared_ptr change; // log_debug("NAME: %1%", name); diff --git a/src/replicator/planetreplicator.cc b/src/replicator/planetreplicator.cc index c8dc306b6..59d43a181 100644 --- a/src/replicator/planetreplicator.cc +++ b/src/replicator/planetreplicator.cc @@ -182,10 +182,6 @@ std::shared_ptr PlanetReplicator::findRemotePath(const underpassconfi fullurl = "https://" + config.planet_server + "/" + cached; remote->parse(fullurl); remote->updatePath(major, minor, index); - if (!config.silent) { - remote->dump(); - } - return remote; }; diff --git a/src/underpass.cc b/src/underpass.cc index f9d2bb50a..c64b34be2 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -224,11 +224,6 @@ main(int argc, char *argv[]) } planetreplicator::PlanetReplicator replicator; - replicator.connectServer("https://" + config.planet_server); - - - std::string fullurl = "https://" + config.planet_server + "/replication/" + StateFile::freq_to_string(config.frequency); - std::vector parts; // Priority boundary multipolygon_t poly; @@ -280,6 +275,7 @@ main(int argc, char *argv[]) exit(-1); } } else if (vm.count("url")) { + replicator.connectServer("https://" + config.planet_server); std::string fullurl = "https://" + config.planet_server + "/replication/" + StateFile::freq_to_string(config.frequency); std::vector parts; boost::split(parts, vm["url"].as(), boost::is_any_of("/")); From 81a39e9829846169b304549fdc5a4f4588e443fd Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 25 Mar 2024 16:17:03 -0300 Subject: [PATCH 7/9] Fixing replication --- src/replicator/replication.cc | 4 ++++ src/replicator/threads.cc | 2 -- src/underpass.cc | 17 +++++++---------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index 85f202109..1b9270296 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -497,12 +497,14 @@ Planet::connectServer(const std::string &planet) ctx.set_verify_mode(ssl::verify_none); // Strip off the https part std::string tmp; + std::cout << "planet: " << planet << std::endl; auto pos = planet.find(":"); if (pos != std::string::npos) { tmp = planet.substr(pos + 3); } else { tmp = planet; } + std::cout << "tmp: " << tmp << std::endl; ssl::context ctx{ssl::context::sslv23_client}; boost::asio::io_context ioc; @@ -801,3 +803,5 @@ Planet::Planet(const RemoteURL &url) // mode: C++ // indent-tabs-mode: nil // End: + + diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 711faf9e5..22d702e16 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -161,7 +161,6 @@ startMonitorChangesets(std::shared_ptr &remote, while (i <= cores/4) { std::rotate(servers.begin(), servers.begin()+1, servers.end()); auto replicationPlanet = std::make_shared(*remote); - replicationPlanet->connectServer(servers.front()); planets.push_back(replicationPlanet); i++; } @@ -292,7 +291,6 @@ startMonitorChanges(std::shared_ptr &remote, while (i <= cores/4) { std::rotate(servers.begin(), servers.begin()+1, servers.end()); auto replicationPlanet = std::make_shared(*remote); - replicationPlanet->connectServer(servers.front()); planets.push_back(replicationPlanet); i++; } diff --git a/src/underpass.cc b/src/underpass.cc index c64b34be2..b672ee9f4 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -223,8 +223,6 @@ main(int argc, char *argv[]) } } - planetreplicator::PlanetReplicator replicator; - // Priority boundary multipolygon_t poly; if (vm.count("boundary")) { @@ -256,6 +254,8 @@ main(int argc, char *argv[]) exit(-1); } + planetreplicator::PlanetReplicator replicator; + auto osmchange = std::make_shared(); // Specify a timestamp used by other options if (vm.count("timestamp")) { @@ -305,13 +305,12 @@ main(int argc, char *argv[]) // Changesets std::thread changesetThread; - if (vm.count("changeseturl")) { - auto changeset = std::make_shared(); + auto changeset = std::make_shared(); + if (vm.count("changeseturl") && !vm.count("osmchanges")) { auto changeseturl = vm["changeseturl"].as(); config.frequency = replication::changeset; std::string fullurl = "https://" + config.planet_server + - "/replication/" + StateFile::freq_to_string(config.frequency) + - "/" + changeseturl + ".osm.gz"; + "/replication/changesets/" + changeseturl + ".osm.gz"; changeset->parse(fullurl); changeset->destdir_base = config.destdir_base; std::vector parts; @@ -320,10 +319,8 @@ main(int argc, char *argv[]) if (!config.silent) { changeset->dump(); } - if (!vm.count("osmchanges")) { - changesetThread = std::thread(replicatorthreads::startMonitorChangesets, - std::ref(changeset), std::ref(*oscboundary), std::ref(config)); - } + changesetThread = std::thread(replicatorthreads::startMonitorChangesets, + std::ref(changeset), std::ref(*oscboundary), config); } // Start processing From e1569a3b362ee74556ef0c669f5deff28c809910 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 25 Mar 2024 16:46:10 -0300 Subject: [PATCH 8/9] Remove debug messages --- src/replicator/replication.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index 1b9270296..33416657b 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -497,15 +497,12 @@ Planet::connectServer(const std::string &planet) ctx.set_verify_mode(ssl::verify_none); // Strip off the https part std::string tmp; - std::cout << "planet: " << planet << std::endl; auto pos = planet.find(":"); if (pos != std::string::npos) { tmp = planet.substr(pos + 3); } else { tmp = planet; } - std::cout << "tmp: " << tmp << std::endl; - ssl::context ctx{ssl::context::sslv23_client}; boost::asio::io_context ioc; From 3bef37eb8f18cf012ae4bde608dd3d7bb8e318cf Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Tue, 26 Mar 2024 14:41:01 -0300 Subject: [PATCH 9/9] Restored previous version of Python DB API --- python/dbapi/api/db.py | 7 +-- python/dbapi/api/raw.py | 100 +++++++++++++++++++------------------- python/dbapi/api/stats.py | 48 +++++------------- 3 files changed, 63 insertions(+), 92 deletions(-) diff --git a/python/dbapi/api/db.py b/python/dbapi/api/db.py index 05418acdc..db647593f 100644 --- a/python/dbapi/api/db.py +++ b/python/dbapi/api/db.py @@ -61,12 +61,9 @@ async def run(self, query, singleObject = False): return result[0] return json.loads((result[0]['result'])) except Exception as e: - # print("\n******* \n" + query + "\n******* \n") + print("\n******* \n" + query + "\n******* \n") print(e) - if singleObject: - return {} - else: - return [] + return None finally: await self.pool.release(conn) return None diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index 783166a96..fdbd347e6 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -29,11 +29,6 @@ def getGeoType(table): return "LineString" return "Node" -def getRelationsGeoType(table): - if table == "ways_poly": - return "MultiPolygon" - return "MultiLineString" - def geoFeaturesQuery( area = None, tags = None, @@ -45,29 +40,21 @@ def geoFeaturesQuery( table = None): geoType = getGeoType(table) - query = "with t_data AS ( \ + query = "with t_ways AS ( \ SELECT '" + geoType + "' as type, " + table + ".osm_id as id, " + table + ".timestamp, geom as geometry, tags, status, hashtags, editor, created_at FROM " + table + " \ LEFT JOIN validation ON validation.osm_id = " + table + ".osm_id \ - LEFT JOIN changesets c ON c.id = " + table + ".changeset" - - if table != "nodes": - query += " UNION \ - SELECT '" + getRelationsGeoType(table) + "' as type, relations.osm_id as id, relations.timestamp, geom as geometry, tags, status, hashtags, editor, created_at FROM relations \ - LEFT JOIN validation ON validation.osm_id = relations.osm_id \ - LEFT JOIN changesets c ON c.id = relations.changeset \ - WHERE (tags->>'type' = '" + getRelationsGeoType(table).lower() + "')" - - query += "), t_results as (select * from t_data WHERE \ + LEFT JOIN changesets c ON c.id = " + table + ".changeset \ + WHERE \ {0} {1} {2} {3} {4} {5} \ ), \ - t_features AS ( \ - SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_results) \ - - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_results \ + t_features AS ( \ + SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_ways) \ + - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways \ ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ as result FROM t_features;".format( - "ST_Intersects(\"geometry\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ", - "AND (" + tagsQueryFilter(tags, "t_data") + ")" if tags else "", - "AND " + hashtagQueryFilter(hashtag, "t_data") if hashtag else "", + "ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ", + "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", + "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", "AND created_at >= {0} AND created_at <= {1}".format(dateFrom, dateTo) if dateFrom and dateTo else "", "AND status = '{0}'".format(status) if (status) else "", "LIMIT " + str(RESULTS_PER_PAGE), @@ -87,33 +74,25 @@ def listAllFeaturesQuery( ): geoType = getGeoType(table) - relationsGeoType = getRelationsGeoType(table) if table == "nodes": osmType = "node" else: osmType = "way" - query = "with t_data AS ( \ + query = "\ + ( \ SELECT '" + osmType + "' as type, '" + geoType + "' as geotype, " + table + ".osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, " + table + ".timestamp, tags, " + table + ".changeset, c.created_at, v.status FROM " + table + " \ LEFT JOIN changesets c ON c.id = " + table + ".changeset \ - LEFT JOIN validation v ON v.osm_id = " + table + ".osm_id" - - if table != "nodes": - query += " UNION \ - SELECT '" + osmType + "' as type, '" + relationsGeoType + "' as geotype, relations.osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, relations.timestamp, tags, relations.changeset, c.created_at, v.status FROM relations \ - LEFT JOIN validation v ON v.osm_id = relations.osm_id \ - LEFT JOIN changesets c ON c.id = relations.changeset \ - WHERE (tags->>'type' = '" + getRelationsGeoType(table).lower() + "')" - - query += ") select * from t_data \ + LEFT JOIN validation v ON v.osm_id = " + table + ".osm_id \ WHERE {0} {1} {2} {3} {4} {5} {6} \ + )\ ".format( "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", "AND status = '{0}'".format(status) if (status) else "", - "AND " + hashtagQueryFilter(hashtag, "t_data") if hashtag else "", + "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", - "AND (" + tagsQueryFilter(tags, "t_data") + ")" if tags else "", + "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", "AND " + orderBy + " IS NOT NULL ORDER BY " + orderBy + " DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}" \ .format(page * RESULTS_PER_PAGE_LIST) if page else ""), ).replace("WHERE 1=1 AND", "WHERE") @@ -173,8 +152,19 @@ def getList( orderBy, page ) - - return self.getPolygonsList( + elif featureType == "polygon": + return self.getPolygonsList( + area, + tags, + hashtag, + dateFrom, + dateTo, + status, + orderBy, + page + ) + else: + return self.getAllList( area, tags, hashtag, @@ -216,8 +206,18 @@ def getFeatures( status, page ) - - return self.getPolygons( + elif featureType == "polygon": + return self.getPolygons( + area, + tags, + hashtag, + dateFrom, + dateTo, + status, + page + ) + else: + return self.getAll( area, tags, hashtag, @@ -362,7 +362,7 @@ async def getPolygonsList( tags, hashtag, status, - orderBy or "id", + orderBy or "ways_poly.osm_id", page or 0, dateFrom, dateTo, @@ -372,7 +372,7 @@ async def getPolygonsList( " UNION ".join([queryPolygons]), dateFrom, dateTo, - orderBy or "id" + orderBy or "osm_id" ) return await self.underpassDB.run(query) @@ -393,7 +393,7 @@ async def getLinesList( tags, hashtag, status, - orderBy or "id", + orderBy or "ways_line.osm_id", page or 0, dateFrom, dateTo, @@ -403,7 +403,7 @@ async def getLinesList( " UNION ".join([queryLines]), dateFrom, dateTo, - orderBy or "id" + orderBy or "osm_id" ) return await self.underpassDB.run(query) @@ -424,7 +424,7 @@ async def getNodesList( tags, hashtag, status, - orderBy or "id", + orderBy or "nodes.osm_id", page or 0, dateFrom, dateTo, @@ -434,7 +434,7 @@ async def getNodesList( " UNION ".join([queryNodes]), dateFrom, dateTo, - orderBy or "id" + orderBy or "osm_id" ) return await self.underpassDB.run(query) @@ -455,7 +455,7 @@ async def getAllList( tags, hashtag, status, - orderBy or "id", + orderBy or "ways_poly.osm_id", page or 0, dateFrom, dateTo, @@ -466,7 +466,7 @@ async def getAllList( tags, hashtag, status, - orderBy or "id", + orderBy or "ways_line.osm_id", page or 0, dateFrom, dateTo, @@ -477,7 +477,7 @@ async def getAllList( tags, hashtag, status, - orderBy or "id", + orderBy or "nodes.osm_id", page or 0, dateFrom, dateTo, @@ -487,6 +487,6 @@ async def getAllList( " UNION ".join([queryPolygons, queryLines, queryNodes]), dateFrom, dateTo, - orderBy or "id" + orderBy or "osm_id" ) return await self.underpassDB.run(query) diff --git a/python/dbapi/api/stats.py b/python/dbapi/api/stats.py index 55d877632..8338f2d7c 100644 --- a/python/dbapi/api/stats.py +++ b/python/dbapi/api/stats.py @@ -45,39 +45,25 @@ async def getCount( select {0}.osm_id from {0} \ left join changesets c on changeset = c.id \ where {1} {2} {3} {4} {5} \ - ".format( + ), \ + count_validated_features as ( \ + select count(distinct(all_features.osm_id)) as count from all_features \ + left join validation v on all_features.osm_id = v.osm_id \ + where v.status = '{6}' \ + ), count_features as (\ + select count(distinct(all_features.osm_id)) as total from all_features \ + ) \ + select count, total from count_validated_features, count_features".format( table, "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", + status ) - - query += " union \ - select relations.osm_id from relations \ - left join changesets c on changeset = c.id \ - where {0} {1} {2} {3} {4} \ - ".format( - "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", - "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", - "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", - "AND (" + tagsQueryFilter(tags, "relations") + ")" if tags else "", - "AND " + hashtagQueryFilter(hashtag, "relations") if hashtag else "", - ) - - query += "), count_validated_features as ( \ - select count(distinct(all_features.osm_id)) as count from all_features \ - left join validation v on all_features.osm_id = v.osm_id \ - where v.status = '{0}' \ - ), count_features as (\ - select count(distinct(all_features.osm_id)) as total from all_features \ - ) \ - select count, total from count_validated_features, count_features".format(status) - else: - query = "WITH counts AS (" - query += "select distinct {0}.osm_id from {0} \ + query = "select count(distinct {0}.osm_id) as count from {0} \ left join changesets c on changeset = c.id \ where {1} {2} {3} {4} {5}".format( table, @@ -87,18 +73,6 @@ async def getCount( "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "" ) - query += " union " - query += "select distinct relations.osm_id from relations \ - left join changesets c on changeset = c.id \ - where {0} {1} {2} {3} {4}".format( - "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", - "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", - "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", - "AND (" + tagsQueryFilter(tags, "relations") + ")" if tags else "", - "AND " + hashtagQueryFilter(hashtag, "relations") if hashtag else "" - ) - query += ") SELECT COUNT(osm_id) FROM counts;" - return(await self.underpassDB.run(query, True)) \ No newline at end of file