v0.2.47..v0.2.48 changeset ChangesetDeriver.cpp

diff --git a/hoot-core/src/main/cpp/hoot/core/algorithms/changeset/ChangesetDeriver.cpp b/hoot-core/src/main/cpp/hoot/core/algorithms/changeset/ChangesetDeriver.cpp
index 41c1830..b4c46b8 100644
--- a/hoot-core/src/main/cpp/hoot/core/algorithms/changeset/ChangesetDeriver.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/algorithms/changeset/ChangesetDeriver.cpp
@@ -30,6 +30,7 @@
 #include <hoot/core/util/Log.h>
 #include <hoot/core/util/ConfigOptions.h>
 #include <hoot/core/util/FileUtils.h>
+#include <hoot/core/util/MapProjector.h>
 
 namespace hoot
 {
@@ -46,13 +47,18 @@ _allowDeletingReferenceFeatures(ConfigOptions().getChangesetAllowDeletingReferen
   if (_from->getProjection()->IsGeographic() == false ||
       _to->getProjection()->IsGeographic() == false)
   {
-    throw IllegalArgumentException("The projections must both be geographic.");
+    throw IllegalArgumentException(
+      "The projections must both be geographic. Input 1: " +
+      MapProjector::toWkt(_from->getProjection()) + ", Input 2: " +
+      MapProjector::toWkt(_to->getProjection()));
   }
 
   _changesByType.clear();
   _changesByType[Change::ChangeType::Create] = 0;
   _changesByType[Change::ChangeType::Modify] = 0;
   _changesByType[Change::ChangeType::Delete] = 0;
+
+  LOG_VARD(_allowDeletingReferenceFeatures);
 }
 
 ChangesetDeriver::~ChangesetDeriver()
@@ -82,6 +88,10 @@ bool ChangesetDeriver::hasMoreChanges()
 
 Change ChangesetDeriver::_nextChange()
 {
+  // TODO: this method is a bit of mess now...refactor into smaller chunks
+
+  LOG_TRACE("Reading next change...");
+
   const long debugId = 6633775;
 
   Change result;
@@ -101,10 +111,12 @@ Change ChangesetDeriver::_nextChange()
     _fromE = _from->readNextElement();
     _numFromElementsParsed++;
     LOG_TRACE("Read next 'from' element: " << _fromE->getElementId());
+    LOG_VART(_fromE->getVersion());
+    LOG_VART(_fromE->getStatus());
   }
   if (!_toE.get() && _to->hasMoreElements())
   {
-    LOG_TRACE("'to' element null and 'to'' has more elements; reading next 'to'' element...");
+    LOG_TRACE("'to' element null and 'to' has more elements; reading next 'to' element...");
     if (Log::getInstance().getLevel() <= Log::Trace && !_to->hasMoreElements())
     {
       LOG_TRACE("Last to element: " << _toE->getElementId());
@@ -112,6 +124,8 @@ Change ChangesetDeriver::_nextChange()
     _toE = _to->readNextElement();
     _numToElementsParsed++;
     LOG_TRACE("Read next 'to' element: " << _toE->getElementId());
+    LOG_VART(_toE->getVersion());
+    LOG_VART(_toE->getStatus());
   }
 
   // if we've run out of "from" elements, create all the remaining elements in "to"
@@ -120,6 +134,8 @@ Change ChangesetDeriver::_nextChange()
     LOG_TRACE(
       "run out of from elements; 'from' element null; 'to' element not null: " <<
       _toE->getElementId() << "; creating 'to' element...");
+    LOG_VART(_toE->getVersion());
+    LOG_VART(_toE->getStatus());
 
     if (Log::getInstance().getLevel() <= Log::Trace && !_to->hasMoreElements())
     {
@@ -144,15 +160,13 @@ Change ChangesetDeriver::_nextChange()
     {
       _numToElementsParsed++;
       LOG_TRACE("Next 'to' element: " << _toE->getElementId());
+      LOG_VART(_toE->getVersion());
+      LOG_VART(_toE->getStatus());
     }
   }
   // if we've run out of "to" elements, delete all the remaining elements in "from"
   else if (_fromE.get() && !_toE.get())
   { 
-    LOG_TRACE(
-      "run out of 'to' elements; to' element null; 'from' element not null: " <<
-      _fromE->getElementId() << "; deleting 'from' element...");
-
     if (Log::getInstance().getLevel() <= Log::Trace && !_from->hasMoreElements())
     {
       LOG_TRACE("Last from element: " << _fromE->getElementId());
@@ -162,7 +176,25 @@ Change ChangesetDeriver::_nextChange()
       LOG_VART(_fromE);
     }
 
-    result = Change(Change::Delete, _fromE);
+    LOG_VART(_fromE->getVersion());
+    LOG_VART(_fromE->getStatus());
+    if (_allowDeletingReferenceFeatures &&
+        !_fromE->getTags().contains(MetadataTags::HootChangeExcludeDelete()))
+    {
+      LOG_TRACE(
+        "run out of 'to' elements; to' element null; 'from' element not null: " <<
+        _fromE->getElementId() << "; deleting 'from' element...");
+      result = Change(Change::Delete, _fromE);
+    }
+    else
+    {
+      // See similar note below where the 'from' element ID is less than the 'to' element ID.
+      LOG_TRACE(
+        "Skipping delete on unknown1 'from' element " << _fromE->getElementId() <<
+        " due to " << ConfigOptions::getChangesetAllowDeletingReferenceFeaturesKey() <<
+        "=false or inclusion of " << MetadataTags::HootChangeExcludeDelete() << " tag...");
+      result = Change(Change::Unknown, _fromE);
+    }
 
     if (_from->hasMoreElements())
     {
@@ -176,6 +208,8 @@ Change ChangesetDeriver::_nextChange()
     {
       _numFromElementsParsed++;
       LOG_TRACE("Next 'from' element: " << _fromE->getElementId());
+      LOG_VART(_fromE->getVersion());
+      LOG_VART(_fromE->getStatus());
     }
   }
   else
@@ -187,14 +221,22 @@ Change ChangesetDeriver::_nextChange()
       LOG_TRACE(
         "skipping identical elements - 'from' element: " << _fromE->getElementId() <<
         " and 'to' element: " << _toE->getElementId() << "...");
+      LOG_VART(_fromE->getVersion());
+      LOG_VART(_fromE->getStatus());
+      LOG_VART(_toE->getVersion());
+      LOG_VART(_toE->getStatus());
 
       if (Log::getInstance().getLevel() <= Log::Trace && !_from->hasMoreElements())
       {
         LOG_TRACE("Last from element: " << _fromE->getElementId());
+        LOG_VART(_fromE->getVersion());
+        LOG_VART(_fromE->getStatus());
       }
       if (Log::getInstance().getLevel() <= Log::Trace && !_to->hasMoreElements())
       {
         LOG_TRACE("Last to element: " << _toE->getElementId());
+        LOG_VART(_toE->getVersion());
+        LOG_VART(_toE->getStatus());
       }
 
       if (Log::getInstance().getLevel() <= Log::Trace &&
@@ -216,6 +258,8 @@ Change ChangesetDeriver::_nextChange()
       { 
         _numToElementsParsed++;
         LOG_TRACE("Next 'to' element: " << _toE->getElementId());
+        LOG_VART(_toE->getVersion());
+        LOG_VART(_toE->getStatus());
       }
 
       if (_from->hasMoreElements())
@@ -230,6 +274,8 @@ Change ChangesetDeriver::_nextChange()
       {
         _numFromElementsParsed++;
         LOG_TRACE("Next 'from' element: " << _fromE->getElementId());
+        LOG_VART(_fromE->getVersion());
+        LOG_VART(_fromE->getStatus());
       }
     }
 
@@ -248,6 +294,8 @@ Change ChangesetDeriver::_nextChange()
       if (Log::getInstance().getLevel() <= Log::Trace && !_to->hasMoreElements())
       {
         LOG_TRACE("Last to element: " << _toE->getElementId());
+        LOG_VART(_toE->getVersion());
+        LOG_VART(_toE->getStatus());
       }
       if (Log::getInstance().getLevel() <= Log::Trace && _toE->getElementId().getId() == debugId)
       {
@@ -268,15 +316,13 @@ Change ChangesetDeriver::_nextChange()
       {
         _numToElementsParsed++;
         LOG_TRACE("Next 'to' element: " << _toE->getElementId());
+        LOG_VART(_toE->getVersion());
+        LOG_VART(_toE->getStatus());
       }
     }
     // if we've run out of "to" elements, delete all the remaining elements in "from"
     else if (_fromE.get() && !_toE.get())
     {
-      LOG_TRACE(
-        "run out of 'to' elements; to' element null; 'from' element not null: " <<
-        _fromE->getElementId() << "; deleting 'from' element...");
-
       if (Log::getInstance().getLevel() <= Log::Trace && !_from->hasMoreElements())
       {
         LOG_TRACE("Last from element: " << _fromE->getElementId());
@@ -286,7 +332,25 @@ Change ChangesetDeriver::_nextChange()
         LOG_VART(_fromE);
       }
 
-      result = Change(Change::Delete, _fromE);
+      LOG_VART(_fromE->getVersion());
+      LOG_VART(_fromE->getStatus());
+      if (_allowDeletingReferenceFeatures &&
+          !_fromE->getTags().contains(MetadataTags::HootChangeExcludeDelete()))
+      {
+        LOG_TRACE(
+          "run out of 'to' elements; to' element null; 'from' element not null: " <<
+          _fromE->getElementId() << "; deleting 'from' element...");
+        result = Change(Change::Delete, _fromE);
+      }
+      else
+      {
+        // See similar note below where the 'from' element ID is less than the 'to' element ID.
+        LOG_TRACE(
+          "Skipping delete on unknown1 'from' element " << _fromE->getElementId() <<
+          " due to " << ConfigOptions::getChangesetAllowDeletingReferenceFeaturesKey() <<
+          "=false or inclusion of " << MetadataTags::HootChangeExcludeDelete() << " tag...");
+        result = Change(Change::Unknown, _fromE);
+      }
 
       if (_from->hasMoreElements())
       {
@@ -300,6 +364,8 @@ Change ChangesetDeriver::_nextChange()
       {
         _numFromElementsParsed++;
         LOG_TRACE("Next 'from' element: " << _fromE->getElementId());
+        LOG_VART(_fromE->getVersion());
+        LOG_VART(_fromE->getStatus());
       }
     }
     else if (_fromE->getElementId() == _toE->getElementId())
@@ -307,14 +373,22 @@ Change ChangesetDeriver::_nextChange()
       LOG_TRACE(
         "'from' element id: " << _fromE->getElementId() << " equals 'to' element id: " <<
         _toE->getElementId() << " modifying 'to' element: ");
+      LOG_VART(_fromE->getVersion());
+      LOG_VART(_fromE->getStatus());
+      LOG_VART(_toE->getVersion());
+      LOG_VART(_toE->getStatus());
 
       if (Log::getInstance().getLevel() <= Log::Trace && !_from->hasMoreElements())
       {
         LOG_TRACE("Last from element: " << _fromE->getElementId());
+        LOG_VART(_fromE->getVersion());
+        LOG_VART(_fromE->getStatus());
       }
       if (Log::getInstance().getLevel() <= Log::Trace && !_to->hasMoreElements())
       {
         LOG_TRACE("Last to element: " << _toE->getElementId());
+        LOG_VART(_toE->getVersion());
+        LOG_VART(_toE->getStatus());
       }
       if (Log::getInstance().getLevel() <= Log::Trace &&
           (_fromE->getElementId().getId() == debugId || _toE->getElementId().getId() == debugId))
@@ -337,6 +411,8 @@ Change ChangesetDeriver::_nextChange()
       {
         _numToElementsParsed++;
         LOG_TRACE("Next 'to' element: " << _toE->getElementId());
+        LOG_VART(_toE->getVersion());
+        LOG_VART(_toE->getStatus());
       }
 
       if (_from->hasMoreElements())
@@ -351,17 +427,12 @@ Change ChangesetDeriver::_nextChange()
       {
         _numFromElementsParsed++;
         LOG_TRACE("Next 'from' element: " << _fromE->getElementId());
+        LOG_VART(_fromE->getVersion());
+        LOG_VART(_fromE->getStatus());
       }
     }
     else if (_fromE->getElementId() < _toE->getElementId())
     {
-      //I don't believe hoot conflation is going to delete reference features (could be wrong)...
-      //only modify them.  So, this is a safety feature.  We need to prove this by conflating a lot
-      //more data.
-      // This logic may need to ref the bounds to be more granular; i.e.  Only prevent deleting
-      // ref features crossing the changeset bounds or split features created from former ref
-      // features crossing the changeset bounds
-
       if (Log::getInstance().getLevel() <= Log::Trace &&
           (_fromE->getElementId().getId() == debugId || _toE->getElementId().getId() == debugId))
       {
@@ -369,26 +440,32 @@ Change ChangesetDeriver::_nextChange()
         LOG_VART(_toE);
       }
 
-      if (_allowDeletingReferenceFeatures ||
-          //this assumes the 'from' dataset was loaded as unknown1
-          (!_allowDeletingReferenceFeatures && _fromE->getStatus() != Status::Unknown1))
+      LOG_VART(_fromE->getVersion());
+      LOG_VART(_fromE->getStatus());
+      LOG_VART(_toE->getVersion());
+      LOG_VART(_toE->getStatus());
+      if ((_allowDeletingReferenceFeatures ||
+          // this assumes the 'from' dataset was loaded as unknown1
+          // TODO: Don't understand the use case for this and its been around for awhile now...need
+          // to define and add a test
+          (!_allowDeletingReferenceFeatures && _fromE->getStatus() != Status::Unknown1)) &&
+          !_fromE->getTags().contains(MetadataTags::HootChangeExcludeDelete()))
       {
         LOG_TRACE(
           "'from' element id: " << _fromE->getElementId() << " less than 'to' element id: " <<
           _toE->getElementId() << "; deleting 'from' element...");
-
         result = Change(Change::Delete, _fromE);
       }
       else
       {
-        //The changeset provider will return no more changes if a null change is returned here.  We
-        //want to force no changes for this particular element, so we're going to use the unknown
-        //change type, which ends up being a no-op.  Skipping an element delete in this situation
-        //minimizes the changeset impact on reference datasets in certain situations.
+        // The changeset provider will return no more changes if a null change is returned here.  We
+        // want to force no changes for this particular element, so we're going to use the unknown
+        // change type, which ends up being a no-op.  Skipping an element delete in this situation
+        // minimizes the changeset impact on reference datasets in certain situations.
         LOG_TRACE(
           "Skipping delete on unknown1 'from' element " << _fromE->getElementId() <<
           " due to " << ConfigOptions::getChangesetAllowDeletingReferenceFeaturesKey() <<
-          "=false...");
+          "=false or inclusion of " << MetadataTags::HootChangeExcludeDelete() << " tag...");
         result = Change(Change::Unknown, _fromE);
       }
 
@@ -404,6 +481,8 @@ Change ChangesetDeriver::_nextChange()
       {
         _numFromElementsParsed++;
         LOG_TRACE("Next 'from' element: " << _fromE->getElementId());
+        LOG_VART(_fromE->getVersion());
+        LOG_VART(_fromE->getStatus());
       }
     }
     else
@@ -411,10 +490,16 @@ Change ChangesetDeriver::_nextChange()
       LOG_TRACE(
         "'from' element id: " << _fromE->getElementId() << " greater than 'to' element id: " <<
         _toE->getElementId() << "; creating 'to' element...");
+      LOG_VART(_fromE->getVersion());
+      LOG_VART(_fromE->getStatus());
+      LOG_VART(_toE->getVersion());
+      LOG_VART(_toE->getStatus());
 
       if (Log::getInstance().getLevel() <= Log::Trace && !_to->hasMoreElements())
       {
         LOG_TRACE("Last to element: " << _toE->getElementId());
+        LOG_VART(_toE->getVersion());
+        LOG_VART(_toE->getStatus());
       }
       if (Log::getInstance().getLevel() <= Log::Trace && _toE->getElementId().getId() == debugId)
       {
@@ -435,6 +520,8 @@ Change ChangesetDeriver::_nextChange()
       {
         _numToElementsParsed++;
         LOG_TRACE("Next 'to' element: " << _toE->getElementId());
+        LOG_VART(_toE->getVersion());
+        LOG_VART(_toE->getStatus());
       }
     }
   }