[generator] Actualized and improved popularity generation.

generator/complex_generator/complex_generator.cpp

@@ -2,8 +2,8 @@
 // Complexes are a hierarchy of interesting geographical features.
 // For the program to work correctly, you need to have in your file system:
 // top_directory
-//      |_ planet.o5m
 //      |_maps_build
+//             |_planet.o5m
 //             |_190223(for example 2019 Feb. 23rd)
 //             |_intermediate_data
 //             |_osm2ft
@@ -76,7 +76,7 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) {
   classificator::Load();
 
   feature::GenerateInfo genInfo;
-  genInfo.m_osmFileName = base::JoinPath(FLAGS_maps_build_path, "..", "planet.o5m");
+  genInfo.m_osmFileName = base::JoinPath(FLAGS_maps_build_path, "planet.o5m");
   genInfo.SetOsmFileType("o5m");
   genInfo.SetNodeStorageType(FLAGS_node_storage);
   genInfo.m_intermediateDir = base::JoinPath(FLAGS_maps_build_path, "intermediate_data");
@@ -85,7 +85,9 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) {
 
   generator::hierarchy::PrintFn print;
   generator::hierarchy::GetMainTypeFn getMainType = generator::hierarchy::GetMainType;
-  std::shared_ptr<generator::FilterInterface> filter = std::make_shared<generator::FilterComplex>();
+  std::shared_ptr<generator::FilterInterface> filter =
+      FLAGS_popularity ? std::make_shared<generator::FilterComplexPopularity::FilterComplex>()
+                       : std::make_shared<generator::FilterComplex>();
 
   if (FLAGS_debug)
   {
@@ -101,18 +103,15 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) {
   generator::RawGenerator rawGenerator(genInfo, threadsCount);
   auto processor = CreateProcessor(generator::ProcessorType::Complex, rawGenerator.GetQueue(),
                                    genInfo.m_intermediateDir, false /* haveBordersForWholeWorld */);
-  generator::cache::IntermediateDataObjectsCache objectsCache;
-  auto const cache = std::make_shared<generator::cache::IntermediateData>(objectsCache, genInfo);
-  auto translator = CreateTranslator(generator::TranslatorType::Complex, processor, cache, genInfo);
+  auto translator = CreateTranslator(generator::TranslatorType::Complex, processor,
+                                     rawGenerator.GetCache()->Clone(), genInfo, FLAGS_popularity);
   auto finalProcessor = std::make_shared<generator::ComplexFinalProcessor>(
       genInfo.m_tmpDir, FLAGS_output, threadsCount);
 
   finalProcessor->SetPrintFunction(print);
   finalProcessor->SetGetMainTypeFunction(getMainType);
   finalProcessor->SetGetNameFunction(generator::hierarchy::GetName);
   finalProcessor->SetFilter(filter);
-  finalProcessor->UseBuildingPartsInfo(
-      genInfo.GetIntermediateFileName(BUILDING_PARTS_MAPPING_FILE));
 
   if (FLAGS_popularity)
   {
@@ -121,11 +120,16 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) {
     // Find directory with *.mwm. Directory FLAGS_maps_build_path must contain directory with *.mwm,
     // whose name must consist of six digits.
     Platform::FilesList files;
-    pl.GetFilesByRegExp(FLAGS_maps_build_path, "[0-9]{6}", files);
+    pl.GetFilesByRegExp(FLAGS_maps_build_path, "^[0-9]{6}$", files);
     CHECK_EQUAL(files.size(), 1, ());
     auto const mwmPath = base::JoinPath(FLAGS_maps_build_path, files[0]);
     finalProcessor->UseCentersEnricher(mwmPath, osm2FtPath);
   }
+  else
+  {
+    finalProcessor->UseBuildingPartsInfo(
+        genInfo.GetIntermediateFileName(BUILDING_PARTS_MAPPING_FILE));
+  }
 
   rawGenerator.GenerateCustom(translator, finalProcessor);
   CHECK(rawGenerator.Execute(), ());

generator/filter_complex.cpp

@@ -3,6 +3,7 @@
 #include "generator/feature_builder.hpp"
 #include "generator/hierarchy.hpp"
 
+#include "indexer/classificator.hpp"
 #include "indexer/ftypes_matcher.hpp"
 
 namespace generator
@@ -19,4 +20,19 @@ bool FilterComplex::IsAccepted(feature::FeatureBuilder const & fb) const
 
   return hierarchy::GetMainType(fb.GetTypes()) != ftype::GetEmptyValue();
 }
+
+std::shared_ptr<FilterInterface> FilterComplexPopularity::Clone() const
+{
+  return std::make_shared<FilterComplexPopularity>();
+}
+
+bool FilterComplexPopularity::IsAccepted(feature::FeatureBuilder const & fb) const
+{
+  if (!fb.IsArea() && !fb.IsPoint())
+    return false;
+
+  auto const t = hierarchy::GetMainType(fb.GetTypes());
+  static auto const buildingPart = classif().GetTypeByPath({"building:part"});
+  return t != ftype::GetEmptyValue() && t != buildingPart;
+}
 }  // namespace generator

generator/filter_complex.hpp

@@ -15,4 +15,14 @@ class FilterComplex : public FilterInterface
 
   bool IsAccepted(feature::FeatureBuilder const & fb) const override;
 };
+
+// The filter will leave only elements for complexes.
+class FilterComplexPopularity : public FilterComplex
+{
+public:
+  // FilterInterface overrides:
+  std::shared_ptr<FilterInterface> Clone() const override;
+
+  bool IsAccepted(feature::FeatureBuilder const & fb) const override;
+};
 }  // namespace generator

generator/final_processor_complex.cpp

@@ -84,26 +84,27 @@ void ComplexFinalProcessor::Process()
       relationBuildingParts = RemoveRelationBuildingParts(fbs);
 
     // This case is second. We build a hierarchy using the geometry of objects and their nesting.
-    auto trees = hierarchy::BuildHierarchy(std::move(fbs), m_getMainType, m_filter);
-
-    // We remove tree roots with tag 'building:part'.
-    base::EraseIf(trees, [](auto const & node) {
-      static auto const & buildingPartChecker = ftypes::IsBuildingPartChecker::Instance();
-      return buildingPartChecker(node->GetData().GetTypes());
-    });
-
-    // We want to transform
-    // building
-    //        |_building-part
-    //                       |_building-part
-    // to
-    // building
-    //        |_building-part
-    //        |_building-part
-    hierarchy::FlattenBuildingParts(trees);
-    // In the end we add objects, which were saved by the collector.
+    auto trees = hierarchy::BuildHierarchy(fbs, m_filter);
+
     if (m_buildingToParts)
     {
+      // We remove tree roots with tag 'building:part'.
+      base::EraseIf(trees, [](auto const & node) {
+        static auto const & buildingPartChecker = ftypes::IsBuildingPartChecker::Instance();
+        return buildingPartChecker(node->GetData().GetTypes());
+      });
+
+      // We want to transform
+      // building
+      //        |_building-part
+      //                       |_building-part
+      // to
+      // building
+      //        |_building-part
+      //        |_building-part
+      hierarchy::FlattenBuildingParts(trees);
+
+      // In the end we add objects, which were saved by the collector.
       hierarchy::AddChildrenTo(trees, [&](auto const & compositeId) {
         auto const & ids = m_buildingToParts->GetBuildingPartsByOutlineId(compositeId);
         std::vector<hierarchy::HierarchyPlace> places;

generator/generator_tests/hierarchy_tests.cpp

@@ -73,7 +73,6 @@ UNIT_CLASS_TEST(TestWithClassificator, ComplexHierarchy_FlattenBuildingParts)
 {
 
   auto trees = generator::hierarchy::BuildHierarchy(MakeTestSet1(),
-                                                    generator::hierarchy::GetMainType,
                                                     std::make_shared<generator::FilterComplex>());
   TEST_EQUAL(trees.size(), 1, ());
   TEST_EQUAL(tree_node::Size(trees[0]), 4, ());
@@ -95,7 +94,6 @@ UNIT_CLASS_TEST(TestWithClassificator, ComplexHierarchy_FlattenBuildingParts)
 UNIT_CLASS_TEST(TestWithClassificator, ComplexHierarchy_AddChildrenTo)
 {
   auto trees = generator::hierarchy::BuildHierarchy(MakeTestSet1(),
-                                                    generator::hierarchy::GetMainType,
                                                     std::make_shared<generator::FilterComplex>());
   TEST_EQUAL(trees.size(), 1, ());
   TEST_EQUAL(tree_node::Size(trees[0]), 4, ());

generator/hierarchy.cpp

@@ -6,6 +6,7 @@
 #include "geometry/rect2d.hpp"
 
 #include "base/assert.hpp"
+#include "base/math.hpp"
 #include "base/stl_helpers.hpp"
 
 #include <algorithm>
@@ -113,16 +114,15 @@ HierarchyLinker::Node::Ptr HierarchyLinker::FindPlaceParent(HierarchyPlace const
     // is contained in a object with tag 'building'. This case is second. We suppose a building part is
     // only inside a building.
     static auto const & buildingChecker = ftypes::IsBuildingChecker::Instance();
-    static auto const & buildingPartChecker = ftypes::IsBuildingPartChecker::Instance();
     auto const & candidate = candidateNode->GetData();
-    if (buildingPartChecker(place.GetTypes()) &&
-        !(buildingChecker(candidate.GetTypes()) || buildingPartChecker(candidate.GetTypes())))
+    if (mainTypeIsBuildingPart(place.GetTypes()) &&
+        !(buildingChecker(candidate.GetTypes()) || mainTypeIsBuildingPart(candidate.GetTypes())))
     {
       return;
     }
 
     // A building part must have children only with 'building:part' type.
-    if (!buildingPartChecker(place.GetTypes()) && buildingPartChecker(candidate.GetTypes()))
+    if (!mainTypeIsBuildingPart(place.GetTypes()) && mainTypeIsBuildingPart(candidate.GetTypes()))
       return;
 
     if (place.GetCompositeId() == candidate.GetCompositeId())
@@ -162,7 +162,7 @@ HierarchyLinker::Node::Ptrs HierarchyLinker::Link()
 }
 
 HierarchyEntryEnricher::HierarchyEntryEnricher(std::string const & osm2FtIdsPath,
-                                             std::string const & countryFullPath)
+                                               std::string const & countryFullPath)
   : m_featureGetter(countryFullPath)
 {
   CHECK(m_osm2FtIds.ReadFromFile(osm2FtIdsPath), (osm2FtIdsPath));
@@ -268,16 +268,26 @@ HierarchyEntry HierarchyLinesBuilder::Transform(HierarchyLinker::Node::Ptr const
   return line;
 }
 
-HierarchyLinker::Node::Ptrs BuildHierarchy(std::vector<feature::FeatureBuilder> && fbs,
-                                           GetMainTypeFn const & getMainType,
+HierarchyLinker::Node::Ptrs BuildHierarchy(std::vector<feature::FeatureBuilder> const & fbs,
                                            std::shared_ptr<FilterInterface> const & filter)
 {
-  base::EraseIf(fbs, [&](auto const & fb) { return !filter->IsAccepted(fb); });
   HierarchyLinker::Node::Ptrs places;
   places.reserve(fbs.size());
-  base::Transform(fbs, std::back_inserter(places), [](auto const & fb) {
-    return tree_node::MakeTreeNode(HierarchyPlace(fb));
-  });
+  for (auto const & fb : fbs)
+  {
+    if (!filter->IsAccepted(fb))
+      continue;
+
+    auto hierarchyPlace = HierarchyPlace(fb);
+    if (!hierarchyPlace.IsPoint() &&
+        base::AlmostEqualAbs(hierarchyPlace.GetArea(), 0.0, std::numeric_limits<double>::epsilon()))
+    {
+      continue;
+    }
+
+    places.emplace_back(tree_node::MakeTreeNode(std::move(hierarchyPlace)));
+  }
+
   auto nodes = HierarchyLinker(std::move(places)).Link();
   // We leave only the trees.
   base::EraseIf(nodes, [](auto const & node) {
@@ -310,7 +320,6 @@ void FlattenBuildingParts(HierarchyLinker::Node::Ptrs & trees)
 {
   for (auto & tree : trees)
   {
-
     CHECK(!tree->HasParent(), ());
 
     std::vector<

generator/hierarchy.hpp

@@ -131,8 +131,7 @@ class HierarchyLinesBuilder
   std::unique_ptr<HierarchyEntryEnricher> m_enricher;
 };
 
-HierarchyLinker::Node::Ptrs BuildHierarchy(std::vector<feature::FeatureBuilder> && fbs,
-                                           GetMainTypeFn const & getMainType,
+HierarchyLinker::Node::Ptrs BuildHierarchy(std::vector<feature::FeatureBuilder> const & fbs,
                                            std::shared_ptr<FilterInterface> const & filter);
 
 // AddChildrenTo adds children to node of tree if fn returns not empty vector of HierarchyPlaces

generator/hierarchy_entry.cpp

@@ -3,6 +3,9 @@
 #include "indexer/feature_utils.hpp"
 #include "indexer/ftypes_matcher.hpp"
 
+#include "geometry/latlon.hpp"
+#include "geometry/mercator.hpp"
+
 #include "coding/string_utf8_multilang.hpp"
 
 #include "base/assert.hpp"
@@ -52,8 +55,9 @@ std::string DebugPrint(HierarchyEntry const & entry)
   ToJSONObject(*obj, "country", entry.m_country);
 
   auto center = base::NewJSONObject();
-  ToJSONObject(*center, "x", entry.m_center.x);
-  ToJSONObject(*center, "y", entry.m_center.y);
+  auto const latLon = mercator::ToLatLon(entry.m_center);
+  ToJSONObject(*center, "lat", latLon.m_lat);
+  ToJSONObject(*center, "lon", latLon.m_lon);
   ToJSONObject(*obj, "center", center);
   return DumpToString(obj);
 }
@@ -82,6 +86,12 @@ uint32_t GetMainType(FeatureParams::Types const & types)
   return it != std::cend(types) ? *it : ftype::GetEmptyValue();
 }
 
+bool mainTypeIsBuildingPart(FeatureParams::Types const & types)
+{
+  static auto const buildingPartType = classif().GetTypeByPath({"building:part"});
+  return GetMainType(types) == buildingPartType;
+}
+
 std::string GetName(StringUtf8Multilang const & str) { return GetRussianName(str); }
 
 std::string HierarchyEntryToCsvString(HierarchyEntry const & entry, char delim)
@@ -99,8 +109,9 @@ coding::CSVReader::Row HierarchyEntryToCsvRow(HierarchyEntry const & entry)
 
   row.emplace_back(parentId);
   row.emplace_back(strings::to_string(entry.m_depth));
-  row.emplace_back(strings::to_string_dac(entry.m_center.x, 7));
-  row.emplace_back(strings::to_string_dac(entry.m_center.y, 7));
+  auto const latLon = mercator::ToLatLon(entry.m_center);
+  row.emplace_back(strings::to_string_dac(latLon.m_lat, 7));
+  row.emplace_back(strings::to_string_dac(latLon.m_lon, 7));
   row.emplace_back(strings::to_string(classif().GetReadableObjectName(entry.m_type)));
   row.emplace_back(strings::to_string(entry.m_name));
   row.emplace_back(strings::to_string(entry.m_country));
@@ -114,8 +125,8 @@ HierarchyEntry HierarchyEntryFromCsvRow(coding::CSVReader::Row const & row)
   auto const & id = row[0];
   auto const & parentId = row[1];
   auto const & depth = row[2];
-  auto const & x = row[3];
-  auto const & y = row[4];
+  auto const & lat = row[3];
+  auto const & lon = row[4];
   auto const & type = row[5];
   auto const & name = row[6];
   auto const & country = row[7];
@@ -126,8 +137,12 @@ HierarchyEntry HierarchyEntryFromCsvRow(coding::CSVReader::Row const & row)
     entry.m_parentId = CompositeId(parentId);
 
   VERIFY(strings::to_size_t(depth, entry.m_depth), (row));
-  VERIFY(strings::to_double(x, entry.m_center.x), (row));
-  VERIFY(strings::to_double(y, entry.m_center.y), (row));
+
+  ms::LatLon latLon;
+  VERIFY(strings::to_double(lat, latLon.m_lat), (row));
+  VERIFY(strings::to_double(lon, latLon.m_lon), (row));
+  entry.m_center = mercator::FromLatLon(latLon);
+
   entry.m_type = classif().GetTypeByReadableObjectName(type);
   entry.m_name = name;
   entry.m_country = country;

generator/hierarchy_entry.hpp

@@ -38,6 +38,7 @@ namespace hierarchy
 static char const kCsvDelimiter = ';';
 
 uint32_t GetMainType(FeatureParams::Types const & types);
+bool mainTypeIsBuildingPart(FeatureParams::Types const & types);
 std::string GetName(StringUtf8Multilang const & str);
 
 coding::CSVReader::Row HierarchyEntryToCsvRow(HierarchyEntry const & entry);

generator/raw_generator.cpp

@@ -35,6 +35,8 @@ void RawGenerator::ForceReloadCache()
 
 std::shared_ptr<FeatureProcessorQueue> RawGenerator::GetQueue() { return m_queue; }
 
+std::shared_ptr<cache::IntermediateData> RawGenerator::GetCache() { return m_cache; }
+
 void RawGenerator::GenerateCountries(bool addAds)
 {
   if (!m_genInfo.m_complexHierarchyFilename.empty())

generator/raw_generator.hpp

@@ -31,6 +31,7 @@ class RawGenerator
   bool Execute();
   std::vector<std::string> const & GetNames() const;
   std::shared_ptr<FeatureProcessorQueue> GetQueue();
+  std::shared_ptr<cache::IntermediateData> GetCache();
   void ForceReloadCache();
 
 private:

generator/translator_complex.cpp

@@ -19,20 +19,24 @@ namespace generator
 {
 TranslatorComplex::TranslatorComplex(std::shared_ptr<FeatureProcessorInterface> const & processor,
                                      std::shared_ptr<cache::IntermediateData> const & cache,
-                                     feature::GenerateInfo const & info)
+                                     feature::GenerateInfo const & info, bool popularity)
   : Translator(processor, cache, std::make_shared<FeatureMaker>(cache->GetCache()))
   , m_tagReplacer(std::make_shared<TagReplacer>(
-                    base::JoinPath(GetPlatform().ResourcesDir(), REPLACED_TAGS_FILE)))
+        base::JoinPath(GetPlatform().ResourcesDir(), REPLACED_TAGS_FILE)))
 {
   auto filters = std::make_shared<FilterCollection>();
   filters->Append(std::make_shared<FilterPlanet>());
-  filters->Append(std::make_shared<FilterComplex>());
+  filters->Append(popularity ? std::make_shared<FilterComplexPopularity>()
+                             : std::make_shared<FilterComplex>());
   filters->Append(std::make_shared<FilterElements>(
       base::JoinPath(GetPlatform().ResourcesDir(), SKIPPED_ELEMENTS_FILE)));
   SetFilter(filters);
 
-  SetCollector(std::make_shared<BuildingPartsCollector>(
-                 info.GetIntermediateFileName(BUILDING_PARTS_MAPPING_FILE), cache->GetCache()));
+  if (!popularity)
+  {
+    SetCollector(std::make_shared<BuildingPartsCollector>(
+        info.GetIntermediateFileName(BUILDING_PARTS_MAPPING_FILE), cache->GetCache()));
+  }
 }
 
 void TranslatorComplex::Preprocess(OsmElement & element)

generator/translator_complex.hpp

@@ -22,7 +22,7 @@ class TranslatorComplex : public Translator
 public:
   explicit TranslatorComplex(std::shared_ptr<FeatureProcessorInterface> const & processor,
                              std::shared_ptr<cache::IntermediateData> const & cache,
-                             feature::GenerateInfo const & info);
+                             feature::GenerateInfo const & info, bool popularity = false);
 
   // TranslatorInterface overrides:
   void Preprocess(OsmElement & element) override;