Addcondition filter

.devcontainer/vcpkg.json

@@ -1,5 +1,6 @@
 {
     "dependencies": [
+        "boost-json",
         "boost-log",
         "boost-program-options",
         "concurrencpp",

CMakeLists.txt

@@ -26,7 +26,7 @@ ENDIF()
 set(Boost_USE_STATIC_LIBS ON)
 set(Boost_USE_STATIC_RUNTIME ON)
 
-find_package(Boost REQUIRED COMPONENTS system log program_options)
+find_package(Boost REQUIRED COMPONENTS system log program_options json)
 find_package(GDAL REQUIRED)
 find_package(geos REQUIRED)
 find_path(OSMIUM_INCLUDE_DIRS "osmium/version.hpp")
@@ -46,7 +46,7 @@ include_directories(${GDAL_INCLUDE_DIRS} ${GEOS_INCLUDE_DIR} ${OSMIUM_INCLUDE_DI
 add_executable(${PROJECT_NAME} ${MAIN_CPP_FILES} ${CPP_FILES})
 
 target_link_libraries(${PROJECT_NAME} PRIVATE ${ICU_LIBRARY}
-  PRIVATE Boost::boost Boost::log Boost::program_options
+  PRIVATE Boost::boost Boost::log Boost::program_options Boost::json
   PRIVATE GEOS::geos
   PRIVATE GDAL::GDAL
   PRIVATE expat bz2

plugins/navteq/converter/Converter.hpp

@@ -204,9 +204,10 @@ class Converter {
   // mutex to protect osmium writer
   static std::mutex osmiumWriterMutex;
 
+  std::filesystem::path executable_path;
+
 private:
   static std::map<std::string, std::string> lang_code_map;
-  std::filesystem::path executable_path;
 };
 
 #endif // CONVERTER_HPP

plugins/navteq/converter/StreetConverter.cpp

@@ -16,7 +16,9 @@
 
 #include "StreetConverter.hpp"
 
+#include <boost/json.hpp>
 #include <boost/log/trivial.hpp>
+#include <fstream>
 #include <osmium/builder/osm_object_builder.hpp>
 #include <osmium/io/writer.hpp>
 #include <osmium/memory/buffer.hpp>
@@ -46,6 +48,10 @@ void StreetConverter::convert(const std::filesystem::path &dir,
   init_cnd_mod(cdms_map, dir);
   init_cnd_dt_mod(cdms_map, dir);
 
+  // patch the conditions with a additional json file
+  // this is a workaround for the informations that are not fixed by HERE yet
+  manipulate_cdms_map(cdms_map, dir);
+
   BOOST_LOG_TRIVIAL(info) << " processing country reference";
   auto cntry_ref_map = init_g_cntry_ref_map(dir);
   auto area_to_govt_code_map = init_g_area_to_govt_code_map(cntry_ref_map, dir);
@@ -73,6 +79,54 @@ void StreetConverter::convert(const std::filesystem::path &dir,
   process_way(dir, data, way_end_points_map, z_level_map, writer);
 }
 
+void StreetConverter::manipulate_cdms_map(
+    std::multimap<uint64_t, cond_type> &cdms_map,
+    const std::filesystem::path &dir) {
+
+  std::string filePrefix = dir.parent_path().filename();
+
+  for (auto const &dir_entry :
+       std::filesystem::directory_iterator{executable_path}) {
+    if (dir_entry.is_directory())
+      continue;
+
+    if (dir_entry.path().extension() != ".json")
+      continue;
+
+    if (dir_entry.path().filename().string().find(filePrefix) ==
+        std::string::npos)
+      continue;
+
+    BOOST_LOG_TRIVIAL(info) << " processing json file: " << dir_entry.path();
+
+    std::ifstream ifs(dir_entry.path());
+    auto jv = boost::json::parse(ifs);
+
+    auto jsonPatch = jv.get_object();
+    if (jsonPatch["cdms"].is_null())
+      return;
+
+    // check if all Ids are in the cdms_map
+    auto cdmsJson = jsonPatch["cdms"].get_object();
+
+    auto condToDelete =
+        boost::json::value_to<std::vector<uint64_t>>(cdmsJson["delete"]);
+
+    for (auto const &condId : condToDelete) {
+      auto it = std::ranges::find_if(cdms_map, [condId](auto const &entry) {
+        return entry.second.cond_id_type == condId;
+      });
+      if (it == cdms_map.end()) {
+        BOOST_LOG_TRIVIAL(error)
+            << "Patch Failed: Conditional modification with id " << condId
+            << " not found in database! See " << dir_entry.path();
+        exit(1);
+      }
+      cdms_map.erase(it);
+    }
+  }
+}
+
 std::map<uint64_t, ushort> StreetConverter::process_alt_steets_route_types(
     const std::filesystem::path &dir) {
 

plugins/navteq/converter/StreetConverter.hpp

@@ -350,6 +350,9 @@ class StreetConverter : public Converter {
                          const std::string &restriction, int direction,
                          bool is_imperial_units, uint64_t max_value);
 
+  void manipulate_cdms_map(std::multimap<uint64_t, cond_type> &cdms_map,
+                           const std::filesystem::path &dir);
+
   // CndMod types (CM)
   static constexpr std::string_view CM_MOD_TYPE = "MOD_TYPE";
   static constexpr std::string_view CM_MOD_VAL = "MOD_VAL";

plugins/navteq/navteq_plugin.cpp

@@ -16,6 +16,9 @@
 #include <ogr_api.h>
 #include <osmium/io/any_input.hpp>
 #include <osmium/io/any_output.hpp>
+#include <osmium/io/output_iterator.hpp>
+#include <osmium/object_pointer_collection.hpp>
+#include <osmium/osm/object_comparisons.hpp>
 #include <ranges>
 
 #include "converter/AdminBoundariesConverter.hpp"
@@ -215,11 +218,11 @@ void navteq_plugin::execute() {
       BOOST_LOG_TRIVIAL(debug) << "Start converting WaterConverter " << dir;
       WaterConverter(executable_path).convert(dir, writer);
     }));
-    // results.emplace_back(executor->submit([this, &dir, &writer]() {
-    //   BOOST_LOG_TRIVIAL(debug)
-    //       << "Start converting HouseNumberConverter " << dir;
-    //   HouseNumberConverter(executable_path).convert(dir, writer);
-    // }));
+    results.emplace_back(executor->submit([this, &dir, &writer]() {
+      BOOST_LOG_TRIVIAL(debug)
+          << "Start converting HouseNumberConverter " << dir;
+      HouseNumberConverter(executable_path).convert(dir, writer);
+    }));
   }
 
   auto allDone =
@@ -253,9 +256,11 @@ void navteq_plugin::sortPBF() {
   osmium::io::File tmpfile(
       output_path.parent_path().append("tmp.pbf").string());
 
-  copyType(writer, tmpfile, osmium::osm_entity_bits::node);
-  copyType(writer, tmpfile, osmium::osm_entity_bits::way);
-  copyType(writer, tmpfile, osmium::osm_entity_bits::relation);
+  for (const auto entity :
+       {osmium::osm_entity_bits::node, osmium::osm_entity_bits::way,
+        osmium::osm_entity_bits::relation}) {
+    copyType(writer, tmpfile, entity);
+  }
 
   writer.close();
 
@@ -264,11 +269,20 @@ void navteq_plugin::sortPBF() {
 
 void navteq_plugin::copyType(osmium::io::Writer &writer, osmium::io::File &file,
                              osmium::osm_entity_bits::type bits) {
-  osmium::io::Reader reader(file, bits);
+  std::vector<osmium::memory::Buffer> data;
+  osmium::ObjectPointerCollection objects;
+
+  osmium::io::Reader reader{file, bits};
   while (osmium::memory::Buffer buffer = reader.read()) {
-    writer(std::move(buffer));
+    osmium::apply(buffer, objects);
+    data.push_back(std::move(buffer));
   }
   reader.close();
+
+  objects.sort(osmium::object_order_type_id_version());
+
+  auto out = osmium::io::make_output_iterator(writer);
+  std::copy(objects.begin(), objects.end(), out);
 }
 
 void navteq_plugin::setBoundingBox(double minX, double minY, double maxX,