Use socket.getaddrinfo instead of socket.gethostbyname

socket.gethostbyname only returns a single IPv4 address. Any additional
IPv4 addresses and all IPv6 addresses are ignored. As a consequence,
hosts without IPv4 addresses are ignored and any additional information
from additional IPv4 addresses is ignored. If IPv4 addresses are rotated
by the implementation of socket.gethostbyname, for example, common with
DNS servers and resolvers, the geolocation information for the host can
change without every call of socket.gethostbyname.

It seems better to consider all addresses, both all IPv4 and IPv6
addresses, and to summarize the geolocation information when only a
single result is required.

Signed-off-by: Matthias-Christian Ott <[email protected]>

mirrormanager2/crawler/continents.py

@@ -80,14 +80,48 @@ def check_continent(config, options, session, categoryUrl):
     hostname = hostname.split(":")[0]
 
     try:
-        hostname = socket.gethostbyname(hostname)
-    except Exception as e:
+        addrinfo = socket.getaddrinfo(hostname, None)
+    except socket.gaierror:
         # Name resolution failed. This means
         # that the base URL is broken.
         raise BrokenBaseUrl() from e
 
-    country = gi.country(hostname).country.iso_code
-    if not country:
+    # Extract the IPv4 and IPv6 address from the tuples returned by getaddrinfo.
+    addresses = set()
+    for family, socktype, proto, canonname, sockaddr in addrinfo:
+        # The GeoIP2 databases contain only information for IPv4 and IPv6
+        # addresses. Therefore, other, unusual address families are ignored.
+        if family == socket.AF_INET:
+            address, port = sockaddr
+            addresses.add(address)
+        elif family == socket.AF_INET6:
+            address, port, flowinfo, scope_id = sockaddr
+            addresses.add(address)
+    # Retrieve the ISO 3166-1 code for each address.
+    countries = []
+    for address in addresses:
+        try:
+            country = gi.country(address)
+        except geoip2.errors.AddressNotFoundError:
+            # If no country object is found for an IPv4 or IPv6 address,
+            # the address is ignored.
+            pass
+        else:
+            iso_code = country.country.iso_code
+            # If the ISO 3166-1 code is not available, the country cannot be
+            # matched to continent. Therefore, the country object is ignored.
+            if iso_code is not None:
+                countries.append(iso_code)
+    # The GeoIP2 databases are not perfect and fully accurate. Therefore,
+    # multiple countries might be returned for hosts with multiple addresses. It
+    # seems best to use the most frequently occuring country if a host has
+    # multiple addresses.
+    country_counter = collections.Counter(countries)
+    if country_counter:
+        # most_common(1) returns a list with one element that is tuple that
+        # consists of the item and its count.
+        country = country_counter.most_common(1)[0][0]
+    else:
         # For hosts with no country in the GeoIP database
         # the default is 'US' as that is where most of
         # Fedora infrastructure systems are running

mirrormanager2/utility/generate_worldmap.py

@@ -37,9 +37,55 @@ def main(config, verbose):
             hostname = urlsplit(hcurl.url)[1]
             if host.id in tracking:
                 continue
+            gir = None
             try:
-                ip = socket.gethostbyname(hostname)
-                gir = gi.city(ip)
+                addrinfo = socket.getaddrinfo(hn, None)
+                # Extract the IPv4 and IPv6 address from the tuples returned by
+                # getaddrinfo.
+                addresses = set()
+                for family, socktype, proto, canonname, sockaddr in addrinfo:
+                    # The GeoIP2 databases contain only information for IPv4 and
+                    # IPv6 addresses. Therefore, other, unusual address families
+                    # are ignored.
+                    if family == socket.AF_INET:
+                        address, port = sockaddr
+                        addresses.add(address)
+                    elif family == socket.AF_INET6:
+                        address, port, flowinfo, scope_id = sockaddr
+                        addresses.add(address)
+                # Retrieve the city object for each address.
+                cities = []
+                for address in addresses:
+                    try:
+                        city = gi.city(address)
+                    except geoip2.errors.AddressNotFoundError:
+                        # If no city object was found for an IPv4 or IPv6
+                        # address, the address is ignored.
+                        pass
+                    else:
+                        # It seems that an empty city record is returned when no
+                        # city was found. If no city has been found for an IPv4
+                        # or IPv6 address, the address is ignored.
+                        if city.city.name is not None:
+                            cities.append(city)
+                # If no city objects were found, the location of a host cannot
+                # be determined.
+                if not cities:
+                    continue
+                city_names = (city.city.name for city in cities)
+                # Only the GeoIP2 Enterprise database has a confidence score for
+                # each city record. Therefore, it seems best to use the most
+                # frequently occuring city if a host has multiple addresses.
+                city_name_counter = collections.Counter(city_names)
+                # most_common(1) returns a list with one element that is tuple
+                # that consists of the item and its count.
+                most_common_city_name = city_name_counter.most_common(1)[0][0]
+                # Find a city object for the most common city name. Any city
+                # object should equivalent for a given city name.
+                for city in cities:
+                    if most_common_city_name == city.city.name:
+                        gir = city
+                        break
             except Exception:
                 continue
             if gir is None: