v0.2.55..v0.2.56 changeset AddressNormalizer.cpp

diff --git a/hoot-core/src/main/cpp/hoot/core/conflate/address/AddressNormalizer.cpp b/hoot-core/src/main/cpp/hoot/core/conflate/address/AddressNormalizer.cpp
index d173ae8..2c220c3 100644
--- a/hoot-core/src/main/cpp/hoot/core/conflate/address/AddressNormalizer.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/conflate/address/AddressNormalizer.cpp
@@ -30,7 +30,8 @@
 // hoot
 #include <hoot/core/util/Log.h>
 #include <hoot/core/conflate/address/LibPostalInit.h>
-#include <hoot/core/conflate/address/AddressTagKeys.h>
+#include <hoot/core/conflate/address/Address.h>
+#include <hoot/core/util/StringUtils.h>
 
 // libpostal
 #include <libpostal/libpostal.h>
@@ -41,11 +42,12 @@ namespace hoot
 AddressNormalizer::AddressNormalizer() :
 _numNormalized(0)
 {
+  _addressTagKeys.reset(new AddressTagKeys());
 }
 
 void AddressNormalizer::normalizeAddresses(const ElementPtr& e)
 {
-  const QSet<QString> addressTagKeys = AddressTagKeys::getInstance().getAddressTagKeys(*e);
+  const QSet<QString> addressTagKeys = _addressTagKeys->getAddressTagKeys(*e);
   LOG_VART(addressTagKeys);
   for (QSet<QString>::const_iterator addressTagKeyItr = addressTagKeys.begin();
        addressTagKeyItr != addressTagKeys.end(); ++addressTagKeyItr)
@@ -84,13 +86,30 @@ void AddressNormalizer::normalizeAddresses(const ElementPtr& e)
 
 QSet<QString> AddressNormalizer::normalizeAddress(const QString& address) const
 {
+  const QString addressToNormalize = address.trimmed().simplified();
+  if (!Address::isStreetIntersectionAddress(addressToNormalize))
+  {
+    return _normalizeAddressWithLibPostal(addressToNormalize);
+  }
+  else
+  {
+    // libpostal doesn't handle intersections very well, so doing intersection normalization with
+    // custom logic
+    return _normalizeAddressIntersection(addressToNormalize);
+  }
+}
+
+QSet<QString> AddressNormalizer::_normalizeAddressWithLibPostal(const QString& address) const
+{
+  LOG_TRACE("Normalizing " << address << " with libpostal...");
+
+  QSet<QString> normalizedAddresses;
+  QString addressCopy = address;
+
   // See note about init of this in AddressParser::parseAddresses.
   LibPostalInit::getInstance();
 
-  LOG_VART(address);
-  const QString addressToNormalize = address.trimmed().simplified();
-  LOG_VART(addressToNormalize);
-  QSet<QString> normalizedAddresses;
+  _prepareAddressForLibPostalNormalization(addressCopy);
 
   size_t num_expansions;
   // specifying a language in the options is optional, but could we get better performance if
@@ -98,14 +117,13 @@ QSet<QString> AddressNormalizer::normalizeAddress(const QString& address) const
   // first, of course)?
   char** expansions =
     libpostal_expand_address(
-      addressToNormalize.toUtf8().data(), libpostal_get_default_options(),
-      &num_expansions);
+      addressCopy.toUtf8().data(), libpostal_get_default_options(), &num_expansions);
   // add all the normalizations libpostal finds as possible addresses
   for (size_t i = 0; i < num_expansions; i++)
   {
     const QString normalizedAddress = QString::fromUtf8(expansions[i]);
     LOG_VART(normalizedAddress);
-    if (_isValidNormalizedAddress(addressToNormalize, normalizedAddress) &&
+    if (_isValidNormalizedAddress(addressCopy, normalizedAddress) &&
         !normalizedAddresses.contains(normalizedAddress))
     {
       normalizedAddresses.insert(normalizedAddress);
@@ -122,6 +140,164 @@ QSet<QString> AddressNormalizer::normalizeAddress(const QString& address) const
   return normalizedAddresses;
 }
 
+QSet<QString> AddressNormalizer::_normalizeAddressIntersection(const QString& address) const
+{
+  LOG_TRACE("Normalizing intersection: " << address << "...");
+
+  const QMap<QString, QString> streetTypeAbbreviationsToFullTypes =
+    Address::getStreetTypeAbbreviationsToFullTypes();
+  const QStringList addressParts =
+    StringUtils::splitOnAny(address, Address::getIntersectionSplitTokens(), 2);
+  LOG_VART(addressParts.size());
+
+  if (addressParts.size() != 2)
+  {
+    throw IllegalArgumentException(
+      "A non-intersection address was passed into street intersection address normalization.");
+  }
+
+  // replace all street type abbreviations in both intersection parts in the address with their full
+  // name counterparts
+
+  QString modifiedAddress;
+  for (int i = 0; i < addressParts.size(); i++)
+  {
+    QString addressPart = addressParts.at(i).trimmed();
+    LOG_VART(addressPart);
+    for (QMap<QString, QString>::const_iterator itr = streetTypeAbbreviationsToFullTypes.begin();
+         itr != streetTypeAbbreviationsToFullTypes.end(); ++itr)
+    {
+      const QString abbrev = itr.key().trimmed();
+      LOG_VART(abbrev);
+      const QString fullType = itr.value().trimmed();
+      LOG_VART(fullType);
+
+      LOG_VART(addressPart.endsWith(abbrev, Qt::CaseInsensitive));
+      if (addressPart.endsWith(abbrev, Qt::CaseInsensitive))
+      {
+        StringUtils::replaceLastIndexOf(addressPart, abbrev, fullType);
+        LOG_VART(addressPart);
+      }
+    }
+    LOG_VART(addressPart);
+
+    modifiedAddress += addressPart.trimmed();
+    if (i == 0)
+    {
+      modifiedAddress += " and ";
+    }
+  }
+  LOG_VART(modifiedAddress);
+
+  // If one of the intersection parts has a street type and the other doesn't we'll copy one street
+  // type over to the other. This isn't foolproof as we could end up giving one of the intersections
+  // an incorrect street type (the actual element's address tags never get modified, though).
+  // However, it does help with address matching and will remain in place unless its found to be
+  // causing harm in some way.
+
+  QStringList modifiedAddressParts =
+    StringUtils::splitOnAny(modifiedAddress, Address::getIntersectionSplitTokens(), 2);
+  assert(modifiedAddressParts.size() == 2);
+  LOG_VART(modifiedAddressParts);
+  QString firstIntersectionPart = modifiedAddressParts[0].trimmed();
+  LOG_VART(firstIntersectionPart);
+  QString secondIntersectionPart = modifiedAddressParts[1].trimmed();
+  LOG_VART(secondIntersectionPart);
+
+  QStringList streetFullTypes;
+  QStringList streetPluralTypes;
+  const QStringList streetFullTypesTemp =
+    Address::getStreetFullTypesToTypeAbbreviations().keys();
+  streetFullTypes = streetFullTypesTemp;
+
+  // Sometimes intersections have plural street types (suffixes). We want to be able to handle those
+  // as well, but we don't want them plural in the final normalized address.
+  for (int i = 0; i < streetFullTypesTemp.size(); i++)
+  {
+    const QString pluralType = streetFullTypesTemp.at(i) + "s";
+    if (!streetFullTypes.contains(pluralType))
+    {
+      streetFullTypes.append(pluralType);
+    }
+    streetPluralTypes.append(pluralType);
+  }
+  LOG_VART(streetPluralTypes);
+
+  // remove any plural suffixes found; TODO: may not need all of this here
+  if (StringUtils::endsWithAny(firstIntersectionPart.trimmed(), streetPluralTypes))
+  {
+    firstIntersectionPart.chop(1);
+    LOG_VART(firstIntersectionPart);
+  }
+  if (StringUtils::endsWithAny(secondIntersectionPart.trimmed(), streetPluralTypes))
+  {
+    secondIntersectionPart.chop(1);
+    LOG_VART(secondIntersectionPart);
+  }
+  QStringList modifiedAddressPartsTemp;
+  for (int i = 0; i < modifiedAddressParts.size(); i++)
+  {
+    QString modifiedAddressPart = modifiedAddressParts.at(i);
+    if (StringUtils::endsWithAny(modifiedAddressPart.trimmed(), streetPluralTypes))
+    {
+      modifiedAddressPart.chop(1);
+      LOG_VART(modifiedAddressPart);
+    }
+    modifiedAddressPartsTemp.append(modifiedAddressPart);
+  }
+  modifiedAddressParts = modifiedAddressPartsTemp;
+  QString firstIntersectionEndingStreetType =
+    StringUtils::endsWithAnyAsStr(firstIntersectionPart.trimmed(), streetFullTypes).trimmed();
+  if (firstIntersectionEndingStreetType.endsWith('s'))
+  {
+    firstIntersectionEndingStreetType.chop(1);
+  }
+  LOG_VART(firstIntersectionEndingStreetType);
+  QString secondIntersectionEndingStreetType =
+    StringUtils::endsWithAnyAsStr(secondIntersectionPart.trimmed(), streetFullTypes).trimmed();
+  if (secondIntersectionEndingStreetType.endsWith('s'))
+  {
+    secondIntersectionEndingStreetType.chop(1);
+  }
+  LOG_VART(secondIntersectionEndingStreetType);
+
+  if (!firstIntersectionEndingStreetType.isEmpty() &&
+      secondIntersectionEndingStreetType.isEmpty())
+  {
+    LOG_VART(modifiedAddressParts[1]);
+    modifiedAddressParts[1] =
+      modifiedAddressParts[1].trimmed() + " " + firstIntersectionEndingStreetType.trimmed();
+    LOG_VART(modifiedAddressParts[1]);
+  }
+  else if (firstIntersectionEndingStreetType.isEmpty() &&
+           !secondIntersectionEndingStreetType.isEmpty())
+  {
+    LOG_VART(modifiedAddressParts[0]);
+    modifiedAddressParts[0] =
+      modifiedAddressParts[0].trimmed() + " " + secondIntersectionEndingStreetType.trimmed();
+    LOG_VART(modifiedAddressParts[0]);
+  }
+  modifiedAddress = modifiedAddressParts[0].trimmed() + " and " + modifiedAddressParts[1].trimmed();
+  LOG_VART(modifiedAddress);
+
+  // go ahead and send it to libpostal to finish out the normalization to avoid duplicating some
+  // code, although it probably won't change any from this point
+  return _normalizeAddressWithLibPostal(modifiedAddress);
+}
+
+void AddressNormalizer::_prepareAddressForLibPostalNormalization(QString& address)
+{
+  LOG_TRACE("Before normalization fix: " << address);
+  LOG_VART(Address::isStreetIntersectionAddress(address));
+  // This is a nasty thing libpostal does where it changes "St" to "Saint" when it should be
+  // "Street".
+  if (address.endsWith("st", Qt::CaseInsensitive) && !Address::isStreetIntersectionAddress(address))
+  {
+    StringUtils::replaceLastIndexOf(address, "st", "Street");
+  }
+  LOG_TRACE("After normalization fix: " << address);
+}
+
 bool AddressNormalizer::_isValidNormalizedAddress(const QString& inputAddress,
                                                   const QString& normalizedAddress)
 {
@@ -131,7 +307,8 @@ bool AddressNormalizer::_isValidNormalizedAddress(const QString& inputAddress,
     return false;
   }
   // This is a bit of hack, but I don't like the way libpostal is turning "St" or "Street" into
-  // "Saint".  Should probably look into configuration of libpostal for a possible fix instead.
+  // "Saint". Should probably look into configuration of libpostal or update it for a possible fix
+  // instead.
   else if (normalizedAddress.endsWith("saint", Qt::CaseInsensitive) &&
            (inputAddress.endsWith("street", Qt::CaseInsensitive) ||
             inputAddress.endsWith("st", Qt::CaseInsensitive)))