v0.2.47..v0.2.48 changeset OsmJsonReader.cpp

diff --git a/hoot-core/src/main/cpp/hoot/core/io/OsmJsonReader.cpp b/hoot-core/src/main/cpp/hoot/core/io/OsmJsonReader.cpp
index e61e6d1..c5520e4 100644
--- a/hoot-core/src/main/cpp/hoot/core/io/OsmJsonReader.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/io/OsmJsonReader.cpp
@@ -35,6 +35,7 @@
 #include <hoot/core/util/HootException.h>
 #include <hoot/core/util/Log.h>
 #include <hoot/core/util/StringUtils.h>
+#include <hoot/core/util/IoUtils.h>
 
 // Boost
 #include <boost/foreach.hpp>
@@ -61,6 +62,7 @@ int OsmJsonReader::logWarnCount = 0;
 
 HOOT_FACTORY_REGISTER(OsmMapReader, OsmJsonReader)
 
+// TODO: implement Configurable to help simplify this
 OsmJsonReader::OsmJsonReader()
   : _defaultStatus(Status::Invalid),
     _useDataSourceIds(true),
@@ -78,7 +80,12 @@ OsmJsonReader::OsmJsonReader()
     _bboxContinue(true),
     _runParallel(ConfigOptions().getJsonReaderHttpBboxParallel()),
     _coordGridSize(ConfigOptions().getJsonReaderHttpBboxMaxSize()),
-    _threadCount(ConfigOptions().getJsonReaderHttpBboxThreadCount())
+    _threadCount(ConfigOptions().getJsonReaderHttpBboxThreadCount()),
+    _bounds(GeometryUtils::envelopeFromConfigString(ConfigOptions().getConvertBoundingBox())),
+    _keepImmediatelyConnectedWaysOutsideBounds(
+      ConfigOptions().getConvertBoundingBoxKeepImmediatelyConnectedWaysOutsideBounds()),
+    _missingNodeCount(0),
+    _missingWayCount(0)
 {
 }
 
@@ -97,8 +104,7 @@ bool OsmJsonReader::isSupported(const QString& url)
   // Is it a file?
   if (isRelativeUrl || isLocalFile)
   {
-    const QString filename = isRelativeUrl ? myUrl.toString() : myUrl.toLocalFile();
-    if (QFile::exists(filename) && url.endsWith(".json", Qt::CaseInsensitive))
+    if (url.endsWith(".json", Qt::CaseInsensitive) && !url.startsWith("http", Qt::CaseInsensitive))
     {
       return true;
     }
@@ -121,6 +127,7 @@ void OsmJsonReader::open(const QString& url)
   {
     _isFile = false;
     _isWeb = false;
+    _path = url;
 
     // Bail out if unsupported
     if (!isSupported(url))
@@ -162,6 +169,11 @@ void OsmJsonReader::close()
 
 void OsmJsonReader::read(const OsmMapPtr& map)
 {
+  // clear node id maps in case the reader is used for multiple files
+  _nodeIdMap.clear();
+  _relationIdMap.clear();
+  _wayIdMap.clear();
+
   _map = map;
   if (_isFile)
   {
@@ -177,9 +189,34 @@ void OsmJsonReader::read(const OsmMapPtr& map)
     _loadJSON(_results[i]);
     _parseOverpassJson();
   }
+  LOG_VARD(_map->getElementCount());
+
+  // See related note in OsmXmlReader::read.
+  if (!_bounds.isNull())
+  {
+    IoUtils::cropToBounds(_map, _bounds, _keepImmediatelyConnectedWaysOutsideBounds);
+    LOG_VARD(StringUtils::formatLargeNumber(_map->getElementCount()));
+  }
+}
+
+void OsmJsonReader::_readToMap()
+{
+  // clear node id maps in case the reader is used for multiple files
+  _nodeIdMap.clear();
+  _relationIdMap.clear();
+  _wayIdMap.clear();
+
+  _map.reset(new OsmMap());
+  _parseOverpassJson();
+  LOG_VARD(_map->getElementCount());
+
+  if (!_bounds.isNull())
+  {
+    IoUtils::cropToBounds(_map, _bounds, _keepImmediatelyConnectedWaysOutsideBounds);
+    LOG_VARD(StringUtils::formatLargeNumber(_map->getElementCount()));
+  }
 }
 
-// Throws HootException on error
 void OsmJsonReader::_loadJSON(const QString& jsonStr)
 {
   QString json(jsonStr);
@@ -223,19 +260,30 @@ void OsmJsonReader::_loadJSON(const QString& jsonStr)
   }
 }
 
+bool OsmJsonReader::isValidJson(const QString& jsonStr)
+{
+  try
+  {
+    _loadJSON(jsonStr);
+  }
+  catch (const HootException&)
+  {
+    return false;
+  }
+  return true;
+}
+
 OsmMapPtr OsmJsonReader::loadFromString(const QString& jsonStr)
 {
   _loadJSON(jsonStr);
-  _map.reset(new OsmMap());
-  _parseOverpassJson();
+  _readToMap();
   return _map;
 }
 
 OsmMapPtr OsmJsonReader::loadFromPtree(const boost::property_tree::ptree &tree)
 {
   _propTree = tree;
-  _map.reset(new OsmMap());
-  _parseOverpassJson();
+  _readToMap();
   return _map;
 }
 
@@ -250,16 +298,17 @@ OsmMapPtr OsmJsonReader::loadFromFile(const QString& path)
   QTextStream instream(&infile);
   QString jsonStr = instream.readAll();
   _loadJSON(jsonStr);
-  _map.reset(new OsmMap());
-  _parseOverpassJson();
+  _readToMap();
   return _map;
 }
 
 void OsmJsonReader::setConfiguration(const Settings& conf)
 {
-  _runParallel = ConfigOptions(conf).getJsonReaderHttpBboxParallel();
-  _coordGridSize = ConfigOptions(conf).getJsonReaderHttpBboxMaxSize();
-  _threadCount = ConfigOptions(conf).getJsonReaderHttpBboxThreadCount();
+  ConfigOptions opts(conf);
+  _runParallel = opts.getJsonReaderHttpBboxParallel();
+  _coordGridSize = opts.getJsonReaderHttpBboxMaxSize();
+  _threadCount = opts.getJsonReaderHttpBboxThreadCount();
+  setBounds(GeometryUtils::envelopeFromConfigString(opts.getConvertBoundingBox()));
 }
 
 void OsmJsonReader::_parseOverpassJson()
@@ -272,7 +321,8 @@ void OsmJsonReader::_parseOverpassJson()
 
   // Make a map, and iterate through all of our elements, adding them
   pt::ptree elements = _propTree.get_child("elements");
-  for (pt::ptree::const_iterator elementIt = elements.begin(); elementIt != elements.end(); ++elementIt)
+  for (pt::ptree::const_iterator elementIt = elements.begin(); elementIt != elements.end();
+       ++elementIt)
   {
     // Type can be node, way, or relation
     string typeStr = elementIt->second.get("type", string("--"));
@@ -304,20 +354,61 @@ void OsmJsonReader::_parseOverpassJson()
   }
 }
 
-void OsmJsonReader::_parseOverpassNode(const pt::ptree &item)
+void OsmJsonReader::_parseOverpassNode(const pt::ptree& item)
 {
+  const long debugId = -25928;
+
   // Get info we need to construct our node
-  long id = -1;
+  long id = item.get("id", id);
+
+  if (_nodeIdMap.contains(id))
+  {
+    throw HootException(
+      QString("Duplicate node id %1 in map %2 encountered.").arg(id).arg(_path));
+  }
+
+  long newId;
   if (_useDataSourceIds)
-    id = item.get("id", id);
+  {
+    newId = id;
+  }
+  else
+  {
+    newId = _map->createNextNodeId();
+  }
+  LOG_VART(id);
+  LOG_VART(newId);
+  _nodeIdMap.insert(id, newId);
+
+  const QString msg = "Reading " + ElementId(ElementType::Node, newId).toString() + "...";
+  if (newId == debugId)
+  {
+    LOG_VARD(msg);
+  }
   else
-    id = _map->createNextNodeId();
+  {
+    LOG_VART(msg);
+  }
 
   double lat = item.get("lat", 0.0);
   double lon = item.get("lon", 0.0);
 
+  long version = ElementData::VERSION_EMPTY;
+  version = item.get("version", version);
+  long changeset = ElementData::CHANGESET_EMPTY;
+  changeset = item.get("changeset", changeset);
+  unsigned int timestamp = ElementData::TIMESTAMP_EMPTY;
+  timestamp = item.get("timestamp", timestamp);
+  std::string user = ElementData::USER_EMPTY.toStdString();
+  user = item.get("user", user);
+  long uid = ElementData::UID_EMPTY;
+  uid = item.get("uid", uid);
+
   // Construct node
-  NodePtr pNode(new Node(_defaultStatus, id, lon, lat, _defaultCircErr));
+  NodePtr pNode(
+    new Node(
+      _defaultStatus, newId, lon, lat, _defaultCircErr, changeset, version, timestamp,
+      QString::fromStdString(user), uid));
 
   // Add tags
   _addTags(item, pNode);
@@ -325,6 +416,8 @@ void OsmJsonReader::_parseOverpassNode(const pt::ptree &item)
   // Add node to map
   _map->addNode(pNode);
 
+  LOG_TRACE("Loaded node: " << pNode);
+
   _numRead++;
   if (_numRead % _statusUpdateInterval == 0)
   {
@@ -332,17 +425,56 @@ void OsmJsonReader::_parseOverpassNode(const pt::ptree &item)
   }
 }
 
-void OsmJsonReader::_parseOverpassWay(const pt::ptree &item)
+void OsmJsonReader::_parseOverpassWay(const pt::ptree& item)
 {
+  const long debugId = -3047;
+
   // Get info we need to construct our way
-  long id = -1;
+  long id = item.get("id", id);
+
+  if (_wayIdMap.contains(id))
+  {
+    throw HootException(
+      QString("Duplicate way id %1 in map %2 encountered.").arg(id).arg(_path));
+  }
+
+  long newId;
   if (_useDataSourceIds)
-    id = item.get("id", id);
+  {
+    newId = id;
+  }
   else
-    id = _map->createNextWayId();
+  {
+    newId = _map->createNextWayId();
+  }
+  _wayIdMap.insert(id, newId);
+
+  const QString msg = "Reading " + ElementId(ElementType::Way, newId).toString() + "...";
+  if (newId == debugId)
+  {
+    LOG_VARD(msg);
+  }
+  else
+  {
+    LOG_VART(msg);
+  }
+
+  long version = ElementData::VERSION_EMPTY;
+  version = item.get("version", version);
+  long changeset = ElementData::CHANGESET_EMPTY;
+  changeset = item.get("changeset", changeset);
+  unsigned int timestamp = ElementData::TIMESTAMP_EMPTY;
+  timestamp = item.get("timestamp", timestamp);
+  std::string user = ElementData::USER_EMPTY.toStdString();
+  user = item.get("user", user);
+  long uid = ElementData::UID_EMPTY;
+  uid = item.get("uid", uid);
 
   // Construct Way
-  WayPtr pWay(new Way(_defaultStatus, id, _defaultCircErr));
+  WayPtr pWay(
+    new Way(
+      _defaultStatus, newId, _defaultCircErr, changeset, version, timestamp,
+      QString::fromStdString(user), uid));
 
   // Add nodes
   if (item.not_found() != item.find("nodes"))
@@ -352,7 +484,32 @@ void OsmJsonReader::_parseOverpassWay(const pt::ptree &item)
     while (nodeIt != nodes.end())
     {
       long v = nodeIt->second.get_value<long>();
-      pWay->addNode(v);
+      LOG_VART(v);
+
+      const bool nodePresent = _nodeIdMap.contains(v);
+      LOG_VART(nodePresent);
+      if (!nodePresent)
+      {
+        _missingNodeCount++;
+        if (logWarnCount < Log::getWarnMessageLimit())
+        {
+          LOG_WARN(
+            "Missing " << ElementId(ElementType::Node, v) << " in " <<
+            ElementId(ElementType::Way, newId) << ".");
+        }
+        else if (logWarnCount == Log::getWarnMessageLimit())
+        {
+          LOG_WARN(className() << ": " << Log::LOG_WARN_LIMIT_REACHED_MESSAGE);
+        }
+        logWarnCount++;
+      }
+      else
+      {
+        long newRef = _nodeIdMap.value(v);
+        LOG_TRACE("Adding way node: " << newRef << "...");
+        pWay->addNode(newRef);
+      }
+
       ++nodeIt;
     }
   }
@@ -363,6 +520,8 @@ void OsmJsonReader::_parseOverpassWay(const pt::ptree &item)
   // Add way to map
   _map->addWay(pWay);
 
+  LOG_TRACE("Loaded way: " << pWay);
+
   _numRead++;
   if (_numRead % _statusUpdateInterval == 0)
   {
@@ -370,17 +529,57 @@ void OsmJsonReader::_parseOverpassWay(const pt::ptree &item)
   }
 }
 
-void OsmJsonReader::_parseOverpassRelation(const pt::ptree &item)
+void OsmJsonReader::_parseOverpassRelation(const pt::ptree& item)
 {
+  const long debugId = 0;
+
   // Get info we need to construct our relation
-  long id = -1;
+  long id = item.get("id", id);
+
+  // See related note in OsmXmlReader::_createRelation.
+//  if (_relationIdMap.contains(id))
+//  {
+//    throw HootException(
+//      QString("Duplicate realtion id %1 in map %2 encountered.").arg(id).arg(_path));
+//  }
+
+  long newId;
   if (_useDataSourceIds)
-    id = item.get("id", id);
+  {
+    newId = id;
+  }
   else
-    id = _map->createNextRelationId();
+  {
+    newId = _map->createNextRelationId();
+  }
+  _relationIdMap.insert(id, newId);
+
+  const QString msg = "Reading " + ElementId(ElementType::Relation, newId).toString() + "...";
+  if (newId == debugId)
+  {
+    LOG_VARD(msg);
+  }
+  else
+  {
+    LOG_VART(msg);
+  }
+
+  long version = ElementData::VERSION_EMPTY;
+  version = item.get("version", version);
+  long changeset = ElementData::CHANGESET_EMPTY;
+  changeset = item.get("changeset", changeset);
+  unsigned int timestamp = ElementData::TIMESTAMP_EMPTY;
+  timestamp = item.get("timestamp", timestamp);
+  std::string user = ElementData::USER_EMPTY.toStdString();
+  user = item.get("user", user);
+  long uid = ElementData::UID_EMPTY;
+  uid = item.get("uid", uid);
 
   // Construct Relation
-  RelationPtr pRelation(new Relation(_defaultStatus, id, _defaultCircErr));
+  RelationPtr pRelation(
+    new Relation(
+      _defaultStatus, newId, _defaultCircErr, "", changeset, version, timestamp,
+      QString::fromStdString(user), uid));
 
   // Add members
   if (item.not_found() != item.find("members"))
@@ -393,9 +592,80 @@ void OsmJsonReader::_parseOverpassRelation(const pt::ptree &item)
       long ref = memberIt->second.get("ref", -1l); // default -1 ?
       string role = memberIt->second.get("role", string(""));
 
-      pRelation->addElement(QString::fromStdString(role),
-                            ElementType::fromString(QString::fromStdString(typeStr)),
-                            ref);
+      bool okToAdd = false;
+      if (typeStr == "node")
+      {
+        const bool memberPresent = _nodeIdMap.contains(ref);
+        if (!memberPresent)
+        {
+          _missingNodeCount++;
+          if (logWarnCount < Log::getWarnMessageLimit())
+          {
+            LOG_WARN(
+              "Missing " << ElementId(ElementType::Node, ref) << " in " <<
+              ElementId(ElementType::Relation, newId) << ".");
+          }
+          else if (logWarnCount == Log::getWarnMessageLimit())
+          {
+            LOG_WARN(className() << ": " << Log::LOG_WARN_LIMIT_REACHED_MESSAGE);
+          }
+          logWarnCount++;
+        }
+        else
+        {
+          okToAdd = true;
+        }
+      }
+      else if (typeStr == "way")
+      {
+        const bool memberPresent = _wayIdMap.contains(ref);
+        if (!memberPresent)
+        {
+          _missingWayCount++;
+          if (logWarnCount < Log::getWarnMessageLimit())
+          {
+            LOG_WARN(
+              "Missing " << ElementId(ElementType::Way, ref) << " in " <<
+              ElementId(ElementType::Relation, newId) << ".");
+          }
+          else if (logWarnCount == Log::getWarnMessageLimit())
+          {
+            LOG_WARN(className() << ": " << Log::LOG_WARN_LIMIT_REACHED_MESSAGE);
+          }
+          logWarnCount++;
+        }
+        else
+        {
+          okToAdd = true;
+        }
+      }
+      else if (typeStr == "relation")
+      {
+        ref = _getRelationId(ref);
+        okToAdd = true;
+      }
+      else
+      {
+        if (logWarnCount < Log::getWarnMessageLimit())
+        {
+          LOG_WARN("Found a relation member with unexpected type: " << typeStr << " in relation ("
+                     << id << ")");
+        }
+        else if (logWarnCount == Log::getWarnMessageLimit())
+        {
+          LOG_WARN(className() << ": " << Log::LOG_WARN_LIMIT_REACHED_MESSAGE);
+        }
+        logWarnCount++;
+        okToAdd = false;
+      }
+
+      if (okToAdd)
+      {
+        LOG_TRACE("Adding relation relation member: " << ref << "...");
+        pRelation->addElement(QString::fromStdString(role),
+                              ElementType::fromString(QString::fromStdString(typeStr)), ref);
+      }
+
       ++memberIt;
     }
   }
@@ -406,6 +676,8 @@ void OsmJsonReader::_parseOverpassRelation(const pt::ptree &item)
   // Add relation to map
   _map->addRelation(pRelation);
 
+  LOG_TRACE("Loaded relation: " << pRelation);
+
   _numRead++;
   if (_numRead % _statusUpdateInterval == 0)
   {
@@ -413,7 +685,30 @@ void OsmJsonReader::_parseOverpassRelation(const pt::ptree &item)
   }
 }
 
-void OsmJsonReader::_addTags(const boost::property_tree::ptree &item, hoot::ElementPtr pElement)
+long OsmJsonReader::_getRelationId(long fileId)
+{
+  long newId;
+  if (_useDataSourceIds)
+  {
+    newId = fileId;
+    _relationIdMap.insert(fileId, newId);
+  }
+  else
+  {
+    if (_relationIdMap.find(fileId) == _relationIdMap.end())
+    {
+      newId = _map->createNextRelationId();
+      _relationIdMap.insert(fileId, newId);
+    }
+    else
+    {
+      newId = _relationIdMap[fileId];
+    }
+  }
+  return newId;
+}
+
+void OsmJsonReader::_addTags(const boost::property_tree::ptree& item, hoot::ElementPtr pElement)
 {
   // Find tags and add them
   if (item.not_found() != item.find("tags"))