-
Notifications
You must be signed in to change notification settings - Fork 74
v0.2.52..v0.2.53 changeset ConflateCmd.cpp
Garret Voltz edited this page Feb 12, 2020
·
1 revision
diff --git a/hoot-core/src/main/cpp/hoot/core/cmd/ConflateCmd.cpp b/hoot-core/src/main/cpp/hoot/core/cmd/ConflateCmd.cpp
index 1b0f283..6db7093 100644
--- a/hoot-core/src/main/cpp/hoot/core/cmd/ConflateCmd.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/cmd/ConflateCmd.cpp
@@ -22,7 +22,7 @@
* This will properly maintain the copyright information. DigitalGlobe
* copyrights will be updated automatically.
*
- * @copyright Copyright (C) 2015, 2017, 2018, 2019 DigitalGlobe (http://www.digitalglobe.com/)
+ * @copyright Copyright (C) 2015, 2017, 2018, 2019, 2020 DigitalGlobe (http://www.digitalglobe.com/)
*/
#include "ConflateCmd.h"
@@ -64,7 +64,10 @@
// Tgs
#include <tgs/System/Timer.h>
+// Qt
#include <QFileInfo>
+#include <QDir>
+#include <QElapsedTimer>
using namespace std;
using namespace Tgs;
@@ -162,6 +165,14 @@ int ConflateCmd::runSimple(QStringList& args)
const QString input2 = args[1];
QString output = args[2];
+ QFileInfo outputInfo(output);
+ LOG_VARD(outputInfo.dir().absolutePath());
+ const bool outputDirSuccess = QDir().mkpath(outputInfo.dir().absolutePath());
+ if (!outputDirSuccess)
+ {
+ throw IllegalArgumentException("Unable to create output path for: " + output);
+ }
+
QString osmApiDbUrl;
if (output.endsWith(".osc.sql"))
{
@@ -204,10 +215,16 @@ int ConflateCmd::runSimple(QStringList& args)
LOG_VART(bytesRead);
QList<QList<SingleStat>> allStats;
- _updateConfigOptionsForAttributeConflation();
- if (isDiffConflate)
+ // The highway.merge.tags.only option only gets used with Attribute Conflation for now, so we'll
+ // use it as the sole identifier for it. If that ever changes, then we'll need a different way
+ // to recognize when AC is occurring.
+ if (ConfigOptions().getHighwayMergeTagsOnly())
+ {
+ _updateConfigOptionsForAttributeConflation();
+ }
+ if (isDiffConflate || ConfigOptions().getHighwayMergeTagsOnly())
{
- _updateConfigOptionsForDifferentialConflation();
+ _disableRoundaboutRemoval();
}
// The number of steps here must be updated as you add/remove job steps in the logic.
@@ -220,6 +237,7 @@ int ConflateCmd::runSimple(QStringList& args)
{
_numTotalTasks++;
}
+
// Only add one task for each set of conflate ops, since NamedOp will create its own task step for
// each op internally.
if (ConfigOptions().getConflatePreOps().size() > 0)
@@ -290,7 +308,7 @@ int ConflateCmd::runSimple(QStringList& args)
map, input2, ConfigOptions().getConflateUseDataSourceIds2(), Status::Unknown2);
currentTask++;
}
- LOG_INFO("Conflating map with " << StringUtils::formatLargeNumber(map->size()) << " elements...");
+ LOG_STATUS("Conflating map with " << StringUtils::formatLargeNumber(map->size()) << " elements...");
double inputBytes = IoSingleStat(IoSingleStat::RChar).value - bytesRead;
LOG_VART(inputBytes);
@@ -329,6 +347,8 @@ int ConflateCmd::runSimple(QStringList& args)
if (ConfigOptions().getConflatePreOps().size() > 0)
{
// apply any user specified pre-conflate operations
+ QElapsedTimer timer;
+ timer.start();
NamedOp preOps(ConfigOptions().getConflatePreOps());
preOps.setProgress(
Progress(
@@ -338,6 +358,9 @@ int ConflateCmd::runSimple(QStringList& args)
stats.append(SingleStat("Apply Pre-Conflate Ops Time (sec)", t.getElapsedAndRestart()));
OsmMapWriterFactory::writeDebugMap(map, "after-pre-ops");
currentTask++;
+ LOG_STATUS(
+ "Conflate pre-operations ran in " + StringUtils::millisecondsToDhms(timer.elapsed()) <<
+ " total.");
}
OsmMapPtr result = map;
@@ -371,6 +394,8 @@ int ConflateCmd::runSimple(QStringList& args)
if (ConfigOptions().getConflatePostOps().size() > 0)
{
// apply any user specified post-conflate operations
+ QElapsedTimer timer;
+ timer.start();
NamedOp postOps(ConfigOptions().getConflatePostOps());
postOps.setProgress(
Progress(
@@ -380,6 +405,9 @@ int ConflateCmd::runSimple(QStringList& args)
stats.append(SingleStat("Apply Post-Conflate Ops Time (sec)", t.getElapsedAndRestart()));
OsmMapWriterFactory::writeDebugMap(result, "after-post-ops");
currentTask++;
+ LOG_STATUS(
+ "Conflate post-operations ran in " + StringUtils::millisecondsToDhms(timer.elapsed()) <<
+ " total.");
}
// doing this after the conflate post ops run, since some invalid reviews are removed by them
@@ -472,16 +500,15 @@ int ConflateCmd::runSimple(QStringList& args)
if (displayStats)
{
+ allStats.append(stats);
if (outputStatsFile.isEmpty())
{
- allStats.append(stats);
QString statsMsg = MapStatsWriter().statsToString(allStats, "\t");
cout << "stats = (stat) OR (input map 1 stat) (input map 2 stat) (output map stat)\n" <<
statsMsg << endl;
}
else
{
- allStats.append(stats);
MapStatsWriter().writeStatsToJson(allStats, outputStatsFile);
cout << "stats = (stat) OR (input map 1 stat) (input map 2 stat) (output map stat) in file: " <<
outputStatsFile << endl;
@@ -509,12 +536,23 @@ float ConflateCmd::_getJobPercentComplete(const int currentTaskNum) const
return (float)currentTaskNum / (float)_numTotalTasks;
}
-void ConflateCmd::_updateConfigOptionsForDifferentialConflation()
+void ConflateCmd::_disableRoundaboutRemoval()
{
+ // This applies to both Attribute and Differential Conflation.
+
// Since Differential throws out all matches, there's no way we can have a bad merge between
// ref/secondary roundabouts. Therefore, no need to replace/remove them. If there's a match, we'll
- // end with no secondary roundabout in the diff output and only the ref roundabout when the diff
- // is applied back to the ref.
+ // end up with no secondary roundabout in the diff output and only the ref roundabout when the
+ // diff is applied back to the ref.
+
+ // Our roundabout handling used with other types of conflation makes no sense for Attribute
+ // Conflation. For the other types of conflation we remove roundabouts b/c hoot isn't very good
+ // at merging them together, so we keep the ref roundabouts and skip conflating them altogether.
+ // Since we always keep the ref geometry in AC, there's no opportunity for bad merging and,
+ // thus, no need to remove them in the first place. So, we'll override that behavior here.
+
+ // If the work for #3442 is done, then we could handle this removal in AttributeConflation.conf
+ // add DifferentialConflation.conf instead of doing it here.
QStringList preConflateOps = ConfigOptions().getConflatePreOps();
const QString removeRoundaboutsClassName = QString::fromStdString(RemoveRoundabouts::className());
@@ -539,25 +577,19 @@ void ConflateCmd::_updateConfigOptionsForAttributeConflation()
// These are some custom adjustments to config opts that must be done for Attribute Conflation.
// There may be a way to eliminate these by adding more custom behavior to the UI.
- // This option only gets used with Attribute Conflation for now, so we'll use it as the sole
- // identifier for it. This could change in the future, but hopefully if that happens, by then
- // we have a better solution for changing these opts in place.
- if (ConfigOptions().getHighwayMergeTagsOnly())
- {
- const QString reviewRelationCritName =
- QString::fromStdString(ReviewRelationCriterion::className());
+ const QString reviewRelationCritName =
+ QString::fromStdString(ReviewRelationCriterion::className());
- // This swaps the logic that removes all reviews with the logic that removes them based on score
- // thresholding.
- if (ConfigOptions().getAttributeConflationAllowReviewsByScore())
- {
- QStringList removeElementsCriteria =
- conf().get(ConfigOptions::getRemoveElementsVisitorElementCriteriaKey()).toStringList();
- removeElementsCriteria.replaceInStrings(
- reviewRelationCritName, QString::fromStdString(ReviewScoreCriterion::className()));
- conf().set(
- ConfigOptions::getRemoveElementsVisitorElementCriteriaKey(), removeElementsCriteria);
- }
+ // This swaps the logic that removes all reviews with the logic that removes them based on score
+ // thresholding.
+ if (ConfigOptions().getAttributeConflationAllowReviewsByScore())
+ {
+ QStringList removeElementsCriteria =
+ conf().get(ConfigOptions::getRemoveElementsVisitorElementCriteriaKey()).toStringList();
+ removeElementsCriteria.replaceInStrings(
+ reviewRelationCritName, QString::fromStdString(ReviewScoreCriterion::className()));
+ conf().set(
+ ConfigOptions::getRemoveElementsVisitorElementCriteriaKey(), removeElementsCriteria);
}
}