Merge pull request #8 from alexey-zakharenkov/improve-sustainability

Improve sustainability

db/create_tables.sql

@@ -38,6 +38,7 @@ CREATE TABLE splitting (
     subregion_ids BIGINT[] NOT NULL,
     mwm_size_est REAL NOT NULL,
     mwm_size_thr INTEGER NOT NULL, -- mwm size threshold in Kb, 4-bytes INTEGER is enough
+    next_level INTEGER NOT NULL,
     geom geometry NOT NULL
 );
-CREATE INDEX splitting_idx ON splitting (osm_border_id, mwm_size_thr);
+CREATE INDEX splitting_idx ON splitting (osm_border_id, mwm_size_thr, next_level);

web/app/auto_split.py

@@ -12,9 +12,12 @@
 class DisjointClusterUnion:
     """Disjoint set union implementation for administrative subregions."""
 
-    def __init__(self, region_id, subregions, mwm_size_thr=None):
+    def __init__(self, region_id, subregions, next_level, mwm_size_thr=None):
+        assert all(s_data['mwm_size_est'] is not None
+                    for s_data in subregions.values())
         self.region_id = region_id
         self.subregions = subregions
+        self.next_level = next_level
         self.mwm_size_thr = mwm_size_thr or MWM_SIZE_THRESHOLD
         self.representatives = {sub_id: sub_id for sub_id in subregions}
         # A cluster is one or more subregions with common borders
@@ -84,7 +87,8 @@ def get_best_cluster_to_join_with(small_cluster_id,
     for subregion_id in subregion_ids:
         for other_subregion_id, length in common_border_matrix[subregion_id].items():
             other_cluster_id = dcu.find_cluster(other_subregion_id)
-            if other_cluster_id != small_cluster_id:
+            if (other_cluster_id != small_cluster_id and
+                    not dcu.clusters[other_cluster_id]['finished']):
                 common_borders[other_cluster_id] += length
     if not common_borders:
         return None
@@ -144,8 +148,10 @@ def find_golden_splitting(conn, border_id, next_level, mwm_size_thr):
                                      next_level, need_cities=True)
     if not subregions:
         return
+    if any(s_data['mwm_size_est'] is None for s_data in subregions.values()):
+        return
 
-    dcu = DisjointClusterUnion(border_id, subregions, mwm_size_thr)
+    dcu = DisjointClusterUnion(border_id, subregions, next_level, mwm_size_thr)
     all_subregion_ids = dcu.get_all_subregion_ids()
     common_border_matrix = calculate_common_border_matrix(conn, all_subregion_ids)
 
@@ -188,6 +194,7 @@ def save_splitting_to_db(conn, dcu: DisjointClusterUnion):
             DELETE FROM {autosplit_table}
             WHERE osm_border_id = {dcu.region_id}
               AND mwm_size_thr = {dcu.mwm_size_thr}
+              AND next_level = {dcu.next_level}
             """)
         for cluster_id, data in dcu.clusters.items():
             subregion_ids = data['subregion_ids']
@@ -196,12 +203,13 @@ def save_splitting_to_db(conn, dcu: DisjointClusterUnion):
             )
             cluster_geometry_sql = get_union_sql(subregion_ids)
             cursor.execute(f"""
-                INSERT INTO {autosplit_table} (osm_border_id, subregion_ids,
-                                               geom, mwm_size_thr, mwm_size_est)
+                INSERT INTO {autosplit_table} (osm_border_id, subregion_ids, geom,
+                                               next_level, mwm_size_thr, mwm_size_est)
                   VALUES (
                     {dcu.region_id},
                     '{subregion_ids_array_str}',
                     ({cluster_geometry_sql}),
+                    {dcu.next_level},
                     {dcu.mwm_size_thr},
                     {data['mwm_size_est']}
                   )

web/app/borders_api.py

@@ -218,16 +218,30 @@ def get_server_configuration():
                    mwm_size_thr=config.MWM_SIZE_THRESHOLD)
 
 
+def prepare_sql_search_string(string):
+    if string.startswith('^'):
+        string = string[1:]
+    else:
+        string = f"%{string}"
+    if string.endswith('$'):
+        string = string[:-1]
+    else:
+        string = f"{string}%"
+    return string
+
+
 @app.route('/search')
 def search():
     query = request.args.get('q')
+    sql_search_string = prepare_sql_search_string(query)
+
     with g.conn.cursor() as cursor:
         cursor.execute(f"""
             SELECT ST_XMin(geom), ST_YMin(geom), ST_XMax(geom), ST_YMax(geom)
             FROM {config.BORDERS_TABLE}
             WHERE name ILIKE %s
             ORDER BY (ST_Area(geography(geom)))
-            LIMIT 1""", (f'%{query}%',)
+            LIMIT 1""", (sql_search_string,)
         )
         if cursor.rowcount > 0:
             rec = cursor.fetchone()
@@ -312,10 +326,10 @@ def join_borders():
     with g.conn.cursor() as cursor:
         try:
             borders_table = config.BORDERS_TABLE
-            free_id = get_free_id()
+            joint_id = get_free_id()
             cursor.execute(f"""
                     UPDATE {borders_table}
-                    SET id = {free_id},
+                    SET id = {joint_id},
                         geom = ST_Union({borders_table}.geom, b2.geom),
                         mwm_size_est = {borders_table}.mwm_size_est + b2.mwm_size_est,
                         count_k = -1
@@ -326,6 +340,26 @@ def join_borders():
         except psycopg2.Error as e:
             g.conn.rollback()
             return jsonify(status=str(e))
+
+        # If joint_id is the only child of its parent, then leave only parent
+        parent_id = get_parent_region_id(g.conn, joint_id)
+        if parent_id is not None:
+            cursor.execute(f"""
+                SELECT count(*) FROM {borders_table} WHERE parent_id = %s
+                """, (parent_id,)
+            )
+            children_cnt = cursor.fetchone()[0]
+            if children_cnt == 1:
+                cursor.execute(f"""
+                    UPDATE {borders_table}
+                    SET mwm_size_est = (SELECT mwm_size_est
+                                        FROM {borders_table}
+                                        WHERE id = %s)
+                    WHERE id = %s
+                    """, (joint_id, parent_id)
+                )
+                cursor.execute(f"DELETE FROM {borders_table} WHERE id = %s",
+                               (joint_id,))
     g.conn.commit()
     return jsonify(status='ok')
 
@@ -413,29 +447,9 @@ def find_osm_borders():
 def copy_from_osm():
     osm_id = int(request.args.get('id'))
     name = request.args.get('name')
-    name_sql = f"'{name}'" if name else "'name'"
-    borders_table = config.BORDERS_TABLE
-    osm_table = config.OSM_TABLE
-    with g.conn.cursor() as cursor:
-        # Check if this id already in use
-        cursor.execute(f"SELECT id FROM {borders_table} WHERE id = %s",
-                       (osm_id,))
-        rec = cursor.fetchone()
-        if rec and rec[0]:
-            return jsonify(status=f"Region with id={osm_id} already exists")
-        cursor.execute(f"""
-            INSERT INTO {borders_table} (id, geom, name, modified, count_k)
-              SELECT osm_id, way, {name_sql}, now(), -1
-              FROM {osm_table}
-              WHERE osm_id = %s
-            """, (osm_id,)
-        )
-    assign_region_to_lowest_parent(osm_id)
-    warnings = []
-    try:
-        update_border_mwm_size_estimation(g.conn, osm_id)
-    except Exception as e:
-        warnings.append(str(e))
+    errors, warnings = copy_region_from_osm(g.conn, osm_id, name)
+    if errors:
+        return jsonify(status='\n'.join(errors))
     g.conn.commit()
     return jsonify(status='ok', warnings=warnings)
 
@@ -983,15 +997,15 @@ def border():
 @app.route('/start_over')
 def start_over():
     try:
-        warnings = create_countries_initial_structure(g.conn)
+        create_countries_initial_structure(g.conn)
     except CountryStructureException as e:
         return jsonify(status=str(e))
 
     autosplit_table = config.AUTOSPLIT_TABLE
     with g.conn.cursor() as cursor:
         cursor.execute(f"DELETE FROM {autosplit_table}")
     g.conn.commit()
-    return jsonify(status='ok', warnings=warnings[:10])
+    return jsonify(status='ok')
 
 
 if __name__ == '__main__':