Store the mirror geolocation in the DB

mirrormanager2/lib/migrations/versions/e67aacbf9f4f_host_geo_coordinates.py

@@ -0,0 +1,26 @@
+"""Host geo coordinates
+
+Revision ID: e67aacbf9f4f
+Revises: b1f02d039900
+Create Date: 2024-07-08 12:08:40.774361
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "e67aacbf9f4f"
+down_revision = "b1f02d039900"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("host", sa.Column("latitude", sa.Float(), nullable=True))
+    op.add_column("host", sa.Column("longitude", sa.Float(), nullable=True))
+
+
+def downgrade():
+    op.drop_column("host", "longitude")
+    op.drop_column("host", "latitude")

mirrormanager2/lib/model.py

@@ -120,6 +120,9 @@ class Host(BASE):
     # This field holds information about the last few crawls.
     # Which protocols were used, crawl duration, ...
     last_crawls = deferred(sa.Column(sa.PickleType(), nullable=True))
+    # GeoIP coordinates
+    latitude = sa.Column(sa.Float(), nullable=True)
+    longitude = sa.Column(sa.Float(), nullable=True)
 
     # Relations
     site = relationship("Site", back_populates="hosts")

mirrormanager2/static/fedora/mirrormanager2.css

@@ -84,6 +84,9 @@ input[type="checkbox"] {
     height: 600px;
     margin-bottom: 1em;
 }
+#map h2 {
+    font-size: 1rem;
+}
 
 table.mm2-table-small {
     width: auto;

mirrormanager2/templates/fedora/_macros.html

@@ -70,7 +70,7 @@ <h4>Export Compliance</h4>
         <div class="row footerlinks">
             <div class="col-12 text-center">
                 <p> Fedora is sponsored by Red Hat. <a href="https://www.redhat.com/en/technologies/linux-platforms/articles/relationship-between-fedora-and-rhel">Learn more about the relationship between Red Hat and Fedora »</a> </p>
-                <div class="py-3"> <a href="https://www.redhat.com/" class="border-0"><img class="rh-logo" height=34px src="{{ url_for('static', filename='redhat.png') }}" alt="Red Hat Logo"></a> </div>
+                <div class="py-3"> <a href="https://www.redhat.com/" class="border-0"><img class="rh-logo" height=34px src="{{ url_for('static', filename='fedora/redhat.png') }}" alt="Red Hat Logo"></a> </div>
                 <p> © 2014 - 2024 Red Hat, Inc. and others. </p>
             </div>
             <div class="col-12 text-center">
@@ -195,7 +195,7 @@ <h4>Export Compliance</h4>
         after="Admin Override for enabling or disabling") }}
     {% endif %}
     {{ render_switch(form.all_sites_can_pull_from_me,
-        after="Allow all mirror sites to pull my content 
+        after="Allow all mirror sites to pull my content
         without explicitly adding them to my list.") }}
     <input type="submit" class="btn btn-primary" value="{{action}}">
     {{ form.csrf_token }}
@@ -279,4 +279,4 @@ <h4>Export Compliance</h4>
     </p>
     {{ form.csrf_token }}
   </form>
-    {% endmacro %}
+    {% endmacro %}

mirrormanager2/templates/fedora/mirrors_location.txt

@@ -0,0 +1,4 @@
+lat	lon	title	description	iconSize	iconOffset	icon
+{% for mirror in mirrors -%}
+{{ mirror.host.latitude }}	{{ mirror.host.longitude }}	<a href='{{ mirror.url }}' rel='noopener noreferrer' target='_blank'>{{ mirror.host_name }}</a>	{{ mirror.host.site.name|default("N/A") }}	24,24	-12,-24	{{ url_for("static", filename="map/f-dot.png") }}
+{% endfor %}

mirrormanager2/utility/generate_worldmap.py

@@ -20,72 +20,36 @@
 from .common import config_option
 
 
-def uniq(input):
-    seen = set()
-    seen_add = seen.add
-    return [x for x in input if not (x in seen or seen_add(x))]
-
-
-def lookup_host_locations(config, gi):
+@click.command()
+@config_option
+def main(config):
+    config = mirrormanager2.lib.read_config(config)
+    gi = geoip2.database.Reader(os.path.join(config["GEOIP_BASE"], "GeoLite2-City.mmdb"))
     db_manager = get_db_manager(config)
-    results = []
     with db_manager.Session() as session:
         embargoed_countries = set(x.upper() for x in config["EMBARGOED_COUNTRIES"])
-        tracking = []
+        tracking = set()
         for hcurl in mirrormanager2.lib.get_host_category_url(session):
-            if hcurl.host_category.host.private or hcurl.host_category.host.site.private:
+            host = hcurl.host_category.host
+            if host.private or host.site.private:
                 continue
             hostname = urlsplit(hcurl.url)[1]
-            if hostname in tracking:
+            if host.id in tracking:
                 continue
             try:
                 ip = socket.gethostbyname(hostname)
                 gir = gi.city(ip)
             except Exception:
                 continue
-            try:
-                name = hcurl.host_category.host.site.name
-            except Exception:
-                name = "N/A"
-            if gir is not None:
-                if gir.country.iso_code in embargoed_countries:
-                    print("skipping " + hostname)
-                    continue
-                t = (hostname, gir.country.iso_code, gir.location.latitude, gir.location.longitude)
-                print("{} {} {} {}".format(*t))
-                results.append([t, name])
-                tracking.append(hostname)
-    return results
-
-
-def doit(output, config):
-    gi = geoip2.database.Reader(os.path.join(config["GEOIP_BASE"], "GeoLite2-City.mmdb"))
-    results = lookup_host_locations(config, gi)
-    if not os.path.isdir(output):
-        os.makedirs(output)
-    marker_url = config.get("APPLICATION_ROOT", "/") + "static/map/f-dot.png"
-    with open(os.path.join(output, "mirrors_location.txt"), "w", encoding="utf-8-sig") as fd:
-        fd.write("lat\tlon\ttitle\tdescription\ticonSize\ticonOffset\ticon\n")
-        for t in results:
-            hostname = t[0][0]
-            lat = t[0][2]
-            lon = t[0][3]
-            fd.write(
-                f"{lat}\t{lon}\t<a href='http://{hostname}/' rel='noopener noreferrer' "
-                f"target='_blank'>{hostname}</a>"
-                f"\t{t[1]}\t21,25\t-10,-25\t{marker_url}\n"
-            )
-
-
-@click.command()
-@config_option
-@click.option(
-    "-o",
-    "--output",
-    type=click.Path(),
-    required=True,
-    help="write output to DIR",
-)
-def main(config, output):
-    config = mirrormanager2.lib.read_config(config)
-    doit(output, config)
+            if gir is None:
+                continue
+            if gir.country.iso_code in embargoed_countries:
+                print(
+                    f"WARNING: host {host.id} ({hostname}) seems to be from an embargoed "
+                    f"country: {gir.country.iso_code}"
+                )
+                continue
+            host.latitude = gir.location.latitude
+            host.longitude = gir.location.longitude
+            tracking.add(host.id)
+        session.commit()

mirrormanager2/views.py

@@ -1,6 +1,7 @@
 import datetime
 import os
 import re
+from urllib.parse import urlsplit
 
 import flask
 from sqlalchemy.exc import SQLAlchemyError
@@ -1202,6 +1203,35 @@ def propagation(repo_id):
     )
 
 
+@views.route("/map/mirrors_location.txt")
+def mirrors_location():
+    results = []
+    tracking = []
+    for hcurl in mmlib.get_host_category_url(DB.session):
+        host = hcurl.host_category.host
+        if host.private or host.site.private:
+            continue
+        if host.latitude is None or host.longitude is None:
+            continue
+        scheme, hostname = urlsplit(hcurl.url)[:2]
+        if not scheme.startswith("http"):
+            continue
+        if hostname in tracking:
+            continue
+        url = f"{scheme}://{hostname}"
+        results.append(
+            {
+                "host": host,
+                "url": url,
+                "host_name": hostname,
+            }
+        )
+        tracking.append(hostname)
+    return flask.render_template("mirrors_location.txt", mirrors=results), {
+        "Content-Type": "text/plain"
+    }
+
+
 @views.route("/crawler/<int:host_id>.log")
 def crawler_log(host_id):
     crawler_log_dir = os.path.join(flask.current_app.config["MM_LOG_DIR"], "crawler")