Merge branch 'ad-freiburg:master' into master

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required (VERSION 3.1)
+cmake_minimum_required (VERSION 3.5)
 set(CMAKE_CXX_STANDARD 11)
 
 project (qlever-petrimaps)

src/qlever-petrimaps/GeomCache.cpp

@@ -7,6 +7,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <cassert>
 #include <cstring>
 #include <fstream>
 #include <iostream>
@@ -384,31 +385,41 @@ double GeomCache::getLoadStatusPercent(bool total) {
   }
 
   if (!total) {
-    return std::atomic<size_t>(_curRow) / static_cast<double>(_totalSize) *
-           100.0;
+    double percent = _curRow / static_cast<double>(_totalSize) * 100.0;
+    return std::min(100.0, percent);
   }
 
   double parsePercent = 95.0;
   double parseIdsPercent = 5.0;
   double totalPercent = 0.0;
   switch (_loadStatusStage) {
     case _LoadStatusStages::Parse:
-      totalPercent = std::atomic<size_t>(_curRow) /
-                     static_cast<double>(_totalSize) * parsePercent;
+      totalPercent = _curRow / static_cast<double>(_totalSize) * parsePercent;
       break;
+
     case _LoadStatusStages::ParseIds:
       totalPercent = parsePercent;
-      totalPercent += std::atomic<size_t>(_curRow) /
-                      static_cast<double>(_totalSize) * parseIdsPercent;
+      totalPercent +=
+          _curRow / static_cast<double>(_totalSize) * parseIdsPercent;
+      break;
+
+    case _LoadStatusStages::FromFile:
+      totalPercent = _curRow / static_cast<double>(_totalSize) * 100.0;
       break;
   }
 
-  return totalPercent;
+  return std::min(100.0, totalPercent);
 }
 
 // _____________________________________________________________________________
 int GeomCache::getLoadStatusStage() { return _loadStatusStage; }
 
+// _____________________________________________________________________________
+size_t GeomCache::getTotalProgress() { return _totalSize; }
+
+// _____________________________________________________________________________
+size_t GeomCache::getCurrentProgress() { return _curRow; }
+
 // _____________________________________________________________________________
 void GeomCache::parseIds(const char *c, size_t size) {
   _loadStatusStage = _LoadStatusStages::ParseIds;
@@ -1036,6 +1047,7 @@ std::string GeomCache::indexHashFromDisk(const std::string &fname) {
 
 // _____________________________________________________________________________
 void GeomCache::fromDisk(const std::string &fname) {
+  _loadStatusStage = _LoadStatusStages::FromFile;
   _points.clear();
   _linePoints.clear();
   _lines.clear();
@@ -1046,27 +1058,73 @@ void GeomCache::fromDisk(const std::string &fname) {
   char tmp[100];
   f.read(tmp, 100);
   tmp[99] = 0;
-
   _indexHash = util::trim(tmp);
 
   size_t numPoints;
+  size_t numLinePoints;
+  size_t numLines;
+  size_t numQidToId;
+  std::streampos posPoints;
+  std::streampos posLinePoints;
+  std::streampos posLines;
+  std::streampos posQidToId;
+  // get total num points
+  // points
   f.read(reinterpret_cast<char *>(&numPoints), sizeof(size_t));
   _points.resize(numPoints);
-  f.read(reinterpret_cast<char *>(&_points[0]),
-         sizeof(util::geo::FPoint) * numPoints);
+  posPoints = f.tellg();
+  f.seekg(sizeof(util::geo::FPoint) * numPoints, f.cur);
+
+  // linePoints
+  f.read(reinterpret_cast<char *>(&numLinePoints), sizeof(size_t));
+  _linePoints.resize(numLinePoints);
+  posLinePoints = f.tellg();
+  f.seekg(sizeof(util::geo::Point<int16_t>) * numLinePoints, f.cur);
+
+  // lines
+  f.read(reinterpret_cast<char *>(&numLines), sizeof(size_t));
+  _lines.resize(numLines);
+  posLines = f.tellg();
+  f.seekg(sizeof(size_t) * numLines, f.cur);
+
+  // qidToId
+  f.read(reinterpret_cast<char *>(&numQidToId), sizeof(size_t));
+  _qidToId.resize(numQidToId);
+  posQidToId = f.tellg();
+  f.seekg(sizeof(IdMapping) * numQidToId, f.cur);
+
+  _totalSize = numPoints + numLinePoints + numLines + numQidToId;
+  _curRow = 0;
 
-  f.read(reinterpret_cast<char *>(&numPoints), sizeof(size_t));
-  _linePoints.resize(numPoints);
-  f.read(reinterpret_cast<char *>(&_linePoints[0]),
-         sizeof(util::geo::Point<int16_t>) * numPoints);
+  // read data from file
+  // points
+  f.seekg(posPoints);
+  for (size_t i = 0; i < numPoints; i++) {
+    f.read(reinterpret_cast<char *>(&_points[i]), sizeof(util::geo::FPoint));
+    _curRow += 1;
+  }
 
-  f.read(reinterpret_cast<char *>(&numPoints), sizeof(size_t));
-  _lines.resize(numPoints);
-  f.read(reinterpret_cast<char *>(&_lines[0]), sizeof(size_t) * numPoints);
+  // linePoints
+  f.seekg(posLinePoints);
+  for (size_t i = 0; i < numLinePoints; i++) {
+    f.read(reinterpret_cast<char *>(&_linePoints[i]),
+           sizeof(util::geo::Point<int16_t>));
+    _curRow += 1;
+  }
 
-  f.read(reinterpret_cast<char *>(&numPoints), sizeof(size_t));
-  _qidToId.resize(numPoints);
-  f.read(reinterpret_cast<char *>(&_qidToId[0]), sizeof(IdMapping) * numPoints);
+  // lines
+  f.seekg(posLines);
+  for (size_t i = 0; i < numLines; i++) {
+    f.read(reinterpret_cast<char *>(&_lines[i]), sizeof(size_t));
+    _curRow += 1;
+  }
+
+  // qidToId
+  f.seekg(posQidToId);
+  for (size_t i = 0; i < numQidToId; i++) {
+    f.read(reinterpret_cast<char *>(&_qidToId[i]), sizeof(IdMapping));
+    _curRow += 1;
+  }
 
   f.close();
 }
@@ -1153,12 +1211,12 @@ std::string GeomCache::requestIndexHash() {
 }
 
 // _____________________________________________________________________________
-void GeomCache::load(const std::string &cacheDir) {
+std::string GeomCache::load(const std::string &cacheDir) {
   std::lock_guard<std::mutex> guard(_m);
 
   if (_ready) {
     auto indexHash = requestIndexHash();
-    if (_indexHash == indexHash) return;
+    if (_indexHash == indexHash) return _indexHash;
     LOG(INFO) << "Loaded index hash (" << _indexHash
               << ") and remote index hash (" << indexHash << ") dont match.";
     _ready = false;
@@ -1196,4 +1254,5 @@ void GeomCache::load(const std::string &cacheDir) {
   }
 
   _ready = true;
+  return _indexHash;
 }

src/qlever-petrimaps/GeomCache.h

@@ -13,6 +13,7 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
+#include <atomic>
 
 #include "qlever-petrimaps/Misc.h"
 #include "util/geo/Geo.h"
@@ -24,8 +25,7 @@ class GeomCache {
   GeomCache() : _backendUrl(""), _curl(0) {}
   explicit GeomCache(const std::string& backendUrl)
       : _backendUrl(backendUrl),
-        _curl(curl_easy_init()) {
-        }
+        _curl(curl_easy_init()) {}
 
   GeomCache& operator=(GeomCache&& o) {
     _backendUrl = o._backendUrl;
@@ -49,7 +49,7 @@ class GeomCache {
     return ready;
   }
 
-  void load(const std::string& cacheFile);
+  std::string load(const std::string& cacheFile);
 
   void request();
   size_t requestSize();
@@ -92,6 +92,8 @@ class GeomCache {
   double getLoadStatusPercent(bool total);
   double getLoadStatusPercent() { return getLoadStatusPercent(false); };
   int getLoadStatusStage();
+  size_t getTotalProgress();
+  size_t getCurrentProgress();
 
  private:
   std::string _backendUrl;
@@ -100,18 +102,17 @@ class GeomCache {
   uint8_t _curByte;
   ID _curId;
   QLEVER_ID_TYPE _maxQid;
-  size_t _curRow, _curUniqueGeom;
+  size_t _totalSize = 0;
+  std::atomic<size_t> _curRow;
+  size_t _curUniqueGeom;
 
-  enum _LoadStatusStages {Parse = 1, ParseIds};
+  enum _LoadStatusStages {Parse = 1, ParseIds, FromFile};
   _LoadStatusStages _loadStatusStage = Parse;
 
   static size_t writeCb(void* contents, size_t size, size_t nmemb, void* userp);
-  static size_t writeCbIds(void* contents, size_t size, size_t nmemb,
-                           void* userp);
-  static size_t writeCbCount(void* contents, size_t size, size_t nmemb,
-                             void* userp);
-  static size_t writeCbString(void* contents, size_t size, size_t nmemb,
-                              void* userp);
+  static size_t writeCbIds(void* contents, size_t size, size_t nmemb, void* userp);
+  static size_t writeCbCount(void* contents, size_t size, size_t nmemb, void* userp);
+  static size_t writeCbString(void* contents, size_t size, size_t nmemb, void* userp);
 
   // Get the right SPARQL query for the given backend.
   const std::string& getQuery(const std::string& backendUrl) const;
@@ -145,7 +146,6 @@ class GeomCache {
   std::fstream _linesF;
   std::fstream _qidToIdF;
 
-  size_t _totalSize = 0;
   size_t _geometryDuplicates = 0;
 
   IdMapping _lastQidToId;