Implemented CSV and TSV export for PostgreSQL.

src/qlever-petrimaps/PetriMapsMain.cpp

@@ -84,31 +84,6 @@ int main(int argc, char** argv) {
     throw std::runtime_error(ss.str());
   }
 
-  // libpqxx test
-  try {
-    // Connect to the database
-    std::string conn = "host=localhost port=5432 dbname=test_database user=test_user password=123456";
-    pqxx::connection c(conn.c_str());
-
-    // Start a transaction. You always work in one.
-    pqxx::work w(c);
-
-    // Return single row of data, number 1.
-    pqxx::row r = w.exec1("SELECT 1");
-
-    // Commit transaction. This is not executed if
-    // exception occured.
-    w.commit();
-
-    // r[0] contains first field. Use as<...>() to
-    // convert from string to other datatype.
-    std::cout << r[0].as<int>() << std::endl;
-  } catch (std::exception const &e) {
-    std::cerr << e.what() << std::endl;
-    return 1;
-  }
-  // ------------
-
   LOG(INFO) << "Starting server...";
   LOG(INFO) << "Max memory is " << maxMemoryGB << " GB...";
   Server serv(maxMemoryGB * 1000000000, cacheDir, cacheLifetime);

src/qlever-petrimaps/SQLCache.cpp

@@ -196,14 +196,12 @@ void SQLCache::loadFromFile(const std::string& fname) {
   size_t numLinePoints;
   size_t numLines;
   size_t numRowIdToResultTableRowId;
-  size_t numGeomColumnIdxs;
   size_t numNonGeomColumnIdxs;
   size_t numResultColumnNames;
   std::streampos posPoints;
   std::streampos posLinePoints;
   std::streampos posLines;
   std::streampos posRowIdToResultTableRowId;
-  std::streampos posGeomColumnIdxs;
   std::streampos posNonGeomColumnIdxs;
   std::streampos posResultColumnNames;
 
@@ -231,12 +229,6 @@ void SQLCache::loadFromFile(const std::string& fname) {
   posRowIdToResultTableRowId = f.tellg();
   f.seekg(sizeof(size_t) * 2 * numRowIdToResultTableRowId, f.cur);
 
-  // _geomColumnIdxs
-  f.read(reinterpret_cast<char*>(&numGeomColumnIdxs), sizeof(size_t));
-  _geomColumnIdxs.resize(numGeomColumnIdxs);
-  posGeomColumnIdxs = f.tellg();
-  f.seekg(sizeof(size_t) * numGeomColumnIdxs, f.cur);
-
   // _nonGeomColumnIdxs
   f.read(reinterpret_cast<char*>(&numNonGeomColumnIdxs), sizeof(size_t));
   _nonGeomColumnIdxs.resize(numNonGeomColumnIdxs);
@@ -288,12 +280,6 @@ void SQLCache::loadFromFile(const std::string& fname) {
     _rowIdToResultTableRowId[rowId] = resultTableRowId;
   }
 
-  // _geomColumnIdxs
-  f.seekg(posGeomColumnIdxs);
-  for (size_t i = 0; i < numGeomColumnIdxs; i++) {
-    f.read(reinterpret_cast<char*>(&_geomColumnIdxs[i]), sizeof(size_t));
-  }
-
   // _nonGeomColumnIdxs
   f.seekg(posNonGeomColumnIdxs);
   for (size_t i = 0; i < numNonGeomColumnIdxs; i++) {
@@ -355,11 +341,6 @@ void SQLCache::serializeToFile(const std::string& fname) const {
     f.write(reinterpret_cast<const char*>(&resultTableRowId), sizeof(size_t));
   }
 
-  // _geomColumnIdxs
-  num = _geomColumnIdxs.size();
-  f.write(reinterpret_cast<const char*>(&num), sizeof(size_t));
-  f.write(reinterpret_cast<const char*>(&_geomColumnIdxs[0]), sizeof(size_t) * num);
-
   // _nonGeomColumnIdxs
   num = _nonGeomColumnIdxs.size();
   f.write(reinterpret_cast<const char*>(&num), sizeof(size_t));
@@ -504,6 +485,36 @@ std::vector<std::map<std::string, std::string>> SQLCache::getAttr() const {
   return attr;
 }
 
+// _____________________________________________________________________________
+std::vector<std::map<std::string, std::string>> SQLCache::getAttrIncludeGeom() const {
+  // Use cursor to process data in batches
+  std::vector<std::map<std::string, std::string>> attr;
+  attr.reserve(_rowCount);
+
+  try {
+    pqxx::work w(*_sqlConn);
+    pqxx::stateless_cursor<pqxx::cursor_base::read_only, pqxx::cursor_base::owned> cursor(w, _finalQuery, "myCursor", false);
+    for (size_t pos = 0; pos < _rowCount; pos += _batchSize) {
+      pqxx::result result = cursor.retrieve(pos, pos + _batchSize);
+      for (const auto &row: result) {
+        std::map<std::string, std::string> rowAttr;
+        for (size_t i = 0; i < _resultColumnNames.size(); i++) {
+          int y = int(i);
+          pqxx::field field = row[y + 1];
+          std::string columnName = _resultColumnNames[y];
+          std::string fieldValue = field.c_str();
+          rowAttr[columnName] = fieldValue;
+        }
+        attr.push_back(rowAttr);
+      }
+    }
+  } catch (const std::exception &e) {
+    throw std::runtime_error(e.what());
+  }
+
+  return attr;
+}
+
 // _____________________________________________________________________________
 pqxx::result SQLCache::processQuery(std::string query, int limit, int offset, bool commit) const {
   query = setQueryLimitOffset(query, limit, offset);

src/qlever-petrimaps/SQLCache.h

@@ -38,6 +38,7 @@ class SQLCache : public GeomCache {
   std::vector<std::pair<ID_TYPE, ID_TYPE>> getRelObjects() const;
   std::map<std::string, std::string> getRowAttr(size_t rowId) const;
   std::vector<std::map<std::string, std::string>> getAttr() const;
+  std::vector<std::map<std::string, std::string>> getAttrIncludeGeom() const;
   void setQuery(std::string query);
   void setQueryHash(std::string queryHash);
 

src/qlever-petrimaps/server/Requestor.h

@@ -39,9 +39,11 @@ class Requestor {
   virtual std::vector<std::pair<std::string, std::string>> requestRow(
       uint64_t) const = 0;
   virtual void requestRows(
-      std::function<
-          void(std::vector<std::vector<std::pair<std::string, std::string>>>)>)
+      std::function<void(std::vector<std::vector<std::pair<std::string, std::string>>>)>)
       const = 0;
+  virtual void requestRowsIncludeGeom(
+      std::function<void(std::vector<std::vector<std::pair<std::string, std::string>>>)>)
+      const {};
 
   void createBboxes(util::geo::FBox& pointBbox, util::geo::DBox& lineBbox);
   void createGrid(util::geo::FBox pointBbox, util::geo::DBox lineBbox);

src/qlever-petrimaps/server/SQLRequestor.cpp

@@ -81,6 +81,23 @@ void SQLRequestor::requestRows(std::function<void(std::vector<std::vector<std::p
   cb(res);
 }
 
+void SQLRequestor::requestRowsIncludeGeom(std::function<void(std::vector<std::vector<std::pair<std::string, std::string>>>)> cb) const {
+  if (!_cache->ready()) {
+    throw std::runtime_error("Geom cache not ready");
+  }
+
+  std::vector<std::map<std::string, std::string>> attr = _cache->getAttrIncludeGeom();
+  std::vector<std::vector<std::pair<std::string, std::string>>> res;
+  res.reserve(attr.size());
+  for (size_t i = 0; i < attr.size(); i++) {
+    std::map<std::string, std::string> rowAttr = attr[i];
+    std::vector<std::pair<std::string, std::string>> pairs = getRowAttrPairs(rowAttr);
+    res.push_back(pairs);
+  }
+
+  cb(res);
+}
+
 std::vector<std::pair<std::string, std::string>> SQLRequestor::getRowAttrPairs(std::map<std::string, std::string> rowAttr) const {
   std::vector<std::pair<std::string, std::string>> pairs;
   pairs.reserve(rowAttr.size());

src/qlever-petrimaps/server/SQLRequestor.h

@@ -24,6 +24,7 @@ class SQLRequestor : public Requestor {
    void request();
    std::vector<std::pair<std::string, std::string>> requestRow(uint64_t row) const;
    void requestRows(std::function<void(std::vector<std::vector<std::pair<std::string, std::string>>>)> cb) const;
+   void requestRowsIncludeGeom(std::function<void(std::vector<std::vector<std::pair<std::string, std::string>>>)> cb) const;
    std::vector<std::pair<std::string, std::string>> getRowAttrPairs(std::map<std::string, std::string> rowAttr) const;
 
  private: