Loading bar for local geoJson file upload and geoJsonHash request

CMakeLists.txt

@@ -37,7 +37,7 @@ if (PNG_FOUND)
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
 endif()
 
-set(CMAKE_CXX_FLAGS_DEBUG          "-Og -g -DLOGLEVEL=3")
+set(CMAKE_CXX_FLAGS_DEBUG          "-g -O0 -DLOGLEVEL=3")#-Og -g -DLOGLEVEL=3")
 set(CMAKE_CXX_FLAGS_MINSIZEREL     "${CMAKE_CXX_FLAGS} -DLOGLEVEL=2")
 set(CMAKE_CXX_FLAGS_RELEASE        "${CMAKE_CXX_FLAGS} -DLOGLEVEL=2")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS} -g -DLOGLEVEL=3")
@@ -63,4 +63,4 @@ add_subdirectory(src)
 install(
   FILES ${CMAKE_BINARY_DIR}/petrimaps DESTINATION bin
   PERMISSIONS OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE
-)
+)

src/qlever-petrimaps/GeoJSONCache.cpp

@@ -14,13 +14,24 @@ std::vector<std::pair<ID_TYPE, ID_TYPE>> GeoJSONCache::getRelObjects() const {
   // Used for GeoJSON, returns all objects as vector<pair<geomID, Row>>
   // geomID starts from 0 ascending, Row = geomID
   std::vector<std::pair<ID_TYPE, ID_TYPE>> objects;
-  objects.reserve(_points.size() + _linePoints.size());
+  objects.reserve(_points.size() + _lines.size());
 
+  size_t idx = 0;
   for (size_t i = 0; i < _points.size(); i++) {
-    objects.push_back({i, i});
+    bool isFirst = std::get<1>(_points[i]);
+    if (isFirst && i > 0) {
+      idx++;
+    }
+    objects.push_back({i, idx});
   }
+
+  idx = 0;
   for (size_t i = 0; i < _lines.size(); i++) {
-    objects.push_back({i + I_OFFSET, i + I_OFFSET});
+    bool isFirst = std::get<1>(_lines[i]);
+    if (isFirst && i > 0) {
+      idx++;
+    }
+    objects.push_back({i + I_OFFSET, idx + I_OFFSET});
   }
 
   return objects;
@@ -131,24 +142,26 @@ void GeoJSONCache::load() {
     auto coords = geom["coordinates"];
     auto properties = feature["properties"];
 
-    LOG(INFO) << geom["type"];
+    LOG(INFO) << type;
 
     // PRIMITIVES
+    // Point
     if (type == "Point") {
       auto point = latLngToWebMerc(FPoint(coords[0], coords[1]));
       if (!pointValid(point)) {
         LOG(INFO) << "[GeomCache] Invalid point found. Skipping...";
         continue;
       }
-      _points.push_back(point);
+      _points.push_back({point, true});
 
       _curUniqueGeom++;
       _attr[numPoints] = properties;
       numPoints++;
-
+
+    // LineString
     } else if (type == "LineString") {
       util::geo::DLine line;
-      line.reserve(2);
+      line.reserve(coords.size());
 
       for (std::vector<float> coord : coords) {
         auto point = latLngToWebMerc(DPoint(coord[0], coord[1]));
@@ -159,18 +172,20 @@ void GeoJSONCache::load() {
         line.push_back(point);
       }
       std::size_t idx = _linePoints.size();
-      _lines.push_back(idx);
+      _lines.push_back({idx, true});
       line = util::geo::densify(line, 200 * 3);
       insertLine(line, false);
 
       _curUniqueGeom++;
       _attr[numLines + I_OFFSET] = properties;
       numLines++;
-
+
+    // Polygon
     } else if (type == "Polygon") {
-      for (auto args : coords) {
+      for (size_t i = 0; i < coords.size(); i++) {
+        auto args = coords[i];
         util::geo::DLine line;
-        line.reserve(2);
+        line.reserve(args.size());
 
         for (std::vector<float> coord : args) {
           auto point = latLngToWebMerc(DPoint(coord[0], coord[1]));
@@ -180,8 +195,10 @@ void GeoJSONCache::load() {
           }
           line.push_back(point);
         }
+
         std::size_t idx = _linePoints.size();
-        _lines.push_back(idx);
+        bool isFirst = i == 0;
+        _lines.push_back({idx, isFirst});
         line = util::geo::densify(line, 200 * 3);
         insertLine(line, true);
       }
@@ -190,24 +207,30 @@ void GeoJSONCache::load() {
       _attr[numLines + I_OFFSET] = properties;
       numLines++;
 
-      // MULTIPART
+    // MULTIPART
+    // MultiPoint
     } else if (type == "MultiPoint") {
-      for (std::vector<float> coord : coords) {
+      for (size_t i = 0; i < coords.size(); i++) {
+        std::vector<float> coord = coords[i];
         auto point = latLngToWebMerc(FPoint(coord[0], coord[1]));
         if (!pointValid(point)) {
           LOG(INFO) << "[GeomCache] Invalid point found. Skipping...";
           continue;
         }
-        _points.push_back(point);
+        bool isFirst = i == 0;
+        _points.push_back({point, isFirst});
       }
 
       _curUniqueGeom++;
       _attr[numPoints] = properties;
       numPoints++;
+
+    // MultiLineString
     } else if (type == "MultiLineString") {
-      for (auto args : coords) {
+      for (size_t i = 0; i < coords.size(); i++) {
+        auto args = coords[i];
         util::geo::DLine line;
-        line.reserve(2);
+        line.reserve(args.size());
 
         for (std::vector<float> coord : args) {
           auto point = latLngToWebMerc(DPoint(coord[0], coord[1]));
@@ -218,20 +241,23 @@ void GeoJSONCache::load() {
           line.push_back(point);
         }
         std::size_t idx = _linePoints.size();
-        _lines.push_back(idx);
+        bool isFirst = i == 0;
+        _lines.push_back({idx, isFirst});
         line = util::geo::densify(line, 200 * 3);
         insertLine(line, false);
       }
 
       _curUniqueGeom++;
       _attr[numLines + I_OFFSET] = properties;
       numLines++;
-
+
+    // MultiPolygon
     } else if (type == "MultiPolygon") {
-      for (auto args1 : coords) {
+      for (size_t i = 0; i < coords.size(); i++) {
+        auto args1 = coords[i];
         for (auto args2 : args1) {
           util::geo::DLine line;
-          line.reserve(2);
+          line.reserve(args2.size());
 
           for (std::vector<float> coord : args2) {
             auto point = latLngToWebMerc(DPoint(coord[0], coord[1]));
@@ -242,7 +268,8 @@ void GeoJSONCache::load() {
             line.push_back(point);
           }
           std::size_t idx = _linePoints.size();
-          _lines.push_back(idx);
+          bool isFirst = i == 0;
+          _lines.push_back({idx, isFirst});
           line = util::geo::densify(line, 200 * 3);
           insertLine(line, true);
         }

src/qlever-petrimaps/GeoJSONCache.h

@@ -22,6 +22,16 @@ class GeoJSONCache : public GeomCache {
     return *this;
   };
 
+  const util::geo::FPoint& getPoint(ID_TYPE id) const {
+    return std::get<0>(_points[id]);
+  }
+  size_t getLine(ID_TYPE id) const {
+    return std::get<0>(_lines[id]);
+  }
+  size_t getLineEnd(ID_TYPE id) const {
+    return id + 1 < _lines.size() ? getLine(id + 1) : _linePoints.size();
+  }
+
   void load();
   void setContent(const std::string& content);
   std::vector<std::pair<ID_TYPE, ID_TYPE>> getRelObjects() const;
@@ -36,6 +46,8 @@ class GeoJSONCache : public GeomCache {
   // Map geomID to map<key, value>
   std::map<size_t, std::map<std::string, std::string>> _attr;
   // ------------------------
+  std::vector<std::tuple<util::geo::FPoint, bool>> _points;
+  std::vector<std::tuple<size_t, bool>> _lines;
 
  protected:
 

src/qlever-petrimaps/GeomCache.h

@@ -40,23 +40,20 @@ class GeomCache {
     return ready;
   }
 
-  const std::vector<util::geo::FPoint>& getPoints() const { return _points; }
+  const virtual util::geo::FPoint& getPoint(ID_TYPE id) const = 0;
+  virtual size_t getLine(ID_TYPE id) const = 0;
+  virtual size_t getLineEnd(ID_TYPE id) const = 0;
+
   const std::vector<util::geo::Point<int16_t>>& getLinePoints() const {
     return _linePoints;
   }
-  const std::vector<size_t>& getLines() const { return _lines; }
-  size_t getLine(ID_TYPE id) const { return _lines[id]; }
-  size_t getLineEnd(ID_TYPE id) const {
-    return id + 1 < _lines.size() ? _lines[id + 1] : _linePoints.size();
-  }
-
-  util::geo::FBox getPointBBox(size_t id) const {
-    return util::geo::getBoundingBox(_points[id]);
-  }
+
   util::geo::DBox getLineBBox(size_t id) const;
 
   double getLoadStatusPercent(bool total);
-  double getLoadStatusPercent() { return getLoadStatusPercent(false); };
+  double getLoadStatusPercent() {
+    return getLoadStatusPercent(false);
+  };
   int getLoadStatusStage();
   size_t getTotalProgress();
   size_t getCurrentProgress();
@@ -72,9 +69,7 @@ class GeomCache {
   mutable std::mutex _m;
   bool _ready = false;
 
-  std::vector<util::geo::FPoint> _points;
   std::vector<util::geo::Point<int16_t>> _linePoints;
-  std::vector<size_t> _lines;
 
   static bool pointValid(const util::geo::FPoint& p);
   static bool pointValid(const util::geo::DPoint& p);

src/qlever-petrimaps/SPARQLCache.h

@@ -27,6 +27,16 @@ class SPARQLCache : public GeomCache {
     return *this;
   };
 
+  const util::geo::FPoint& getPoint(ID_TYPE id) const {
+    return _points[id];
+  }
+  size_t getLine(ID_TYPE id) const {
+    return _lines[id];
+  }
+  size_t getLineEnd(ID_TYPE id) const {
+    return id + 1 < _lines.size() ? getLine(id + 1) : _linePoints.size();
+  }
+
   std::string load(const std::string& cacheFile);
   void request();
   size_t requestSize();
@@ -83,6 +93,8 @@ class SPARQLCache : public GeomCache {
   IdMapping _lastQidToId;
 
   std::vector<IdMapping> _qidToId;
+  std::vector<util::geo::FPoint> _points;
+  std::vector<size_t> _lines;
 
   std::string _dangling, _prev, _raw;
   ParseState _state;

src/qlever-petrimaps/server/Requestor.cpp

@@ -34,18 +34,14 @@ void Requestor::createBboxes(util::geo::FBox& pointBbox,
 
 #pragma omp parallel for num_threads(NUM_THREADS) schedule(static)
   for (size_t t = 0; t < NUM_THREADS; t++) {
-    for (size_t i = batch * t; i < batch * (t + 1) && i < _objects.size();
-         i++) {
+    for (size_t i = batch * t; i < batch * (t + 1) && i < _objects.size(); i++) {
       auto geomId = _objects[i].first;
-
       if (geomId < I_OFFSET) {
         auto pId = geomId;
-        pointBoxes[t] =
-            util::geo::extendBox(_cache->getPoints()[pId], pointBoxes[t]);
+        pointBoxes[t] = util::geo::extendBox(_cache->getPoint(pId), pointBoxes[t]);
       } else if (geomId < std::numeric_limits<ID_TYPE>::max()) {
         auto lId = geomId - I_OFFSET;
-        lineBoxes[t] =
-            util::geo::extendBox(_cache->getLineBBox(lId), lineBoxes[t]);
+        lineBoxes[t] = util::geo::extendBox(_cache->getLineBBox(lId), lineBoxes[t]);
         numLines[t]++;
       }
     }
@@ -126,12 +122,12 @@ void Requestor::createGrid(util::geo::FBox pointBbox,
           for (size_t m = 0; m < clusterI; m++) {
             const auto& p = _objects[i - m];
             auto geomId = p.first;
-            _pgrid.add(_cache->getPoints()[geomId], j);
+            _pgrid.add(_cache->getPoint(geomId), j);
             _clusterObjects.push_back({i - m, {m, clusterI}});
             j++;
           }
         } else {
-          _pgrid.add(_cache->getPoints()[geomId], i);
+          _pgrid.add(_cache->getPoint(geomId), i);
         }
 
         // every 100000 objects, check memory...
@@ -293,7 +289,7 @@ const ResObj Requestor::getNearest(util::geo::DPoint rp, double rad, double res,
           size_t cid = i - _objects.size();
           p = clusterGeom(cid, res);
         } else {
-          p = _cache->getPoints()[_objects[i].first];
+          p = _cache->getPoint(_objects[i].first);
         }
 
         if (!util::geo::contains(p, fbox)) continue;
@@ -564,13 +560,13 @@ util::geo::MultiPoint<float> Requestor::geomPointGeoms(size_t oid,
   for (size_t i = oid;
        i < _objects.size() && _objects[i].second == _objects[oid].second; i++) {
     if (_objects[oid].first >= I_OFFSET) continue;
-    points.push_back(_cache->getPoints()[_objects[i].first]);
+    points.push_back(_cache->getPoint(_objects[i].first));
   }
 
   for (size_t i = oid - 1;
        i < _objects.size() && _objects[i].second == _objects[oid].second; i--) {
     if (_objects[oid].first >= I_OFFSET) continue;
-    points.push_back(_cache->getPoints()[_objects[i].first]);
+    points.push_back(_cache->getPoint(_objects[i].first));
   }
 
   return points;
@@ -602,7 +598,7 @@ util::geo::MultiPolygon<double> Requestor::geomPolyGeoms(size_t oid,
 // _____________________________________________________________________________
 util::geo::FPoint Requestor::clusterGeom(size_t cid, double res) const {
   size_t oid = _clusterObjects[cid].first;
-  const auto& pp = _cache->getPoints()[_objects[oid].first];
+  const auto& pp = _cache->getPoint(_objects[oid].first);
 
   if (res < 0) return {pp};
 

src/qlever-petrimaps/server/Requestor.h

@@ -64,7 +64,7 @@ class Requestor {
   }
 
   const util::geo::FPoint& getPoint(ID_TYPE id) const {
-    return _cache->getPoints()[id];
+    return _cache->getPoint(id);
   }
 
   size_t getLine(ID_TYPE id) const { return _cache->getLine(id); }

src/qlever-petrimaps/server/Server.cpp

@@ -74,6 +74,7 @@ util::http::Answer Server::handle(const util::http::Req& req, int con) const {
   try {
     Params params;
     auto cmd = parseUrl(req.url, req.payload, &params);
+    LOG(INFO) << "[SERVER] CMD: " << cmd;
 
     if (cmd == "/") {
       a = util::http::Answer(