Merge remote-tracking branch 'DoFabien/master'

rio_rgbify/mbtiler.py

@@ -322,6 +322,8 @@ def run(self, processes=4):
 
         # create a connection to the mbtiles file
         conn = sqlite3.connect(self.outpath)
+        # Wall mode : Speedup by 10 the speed of writing in the database
+        conn.execute('pragma journal_mode=wal')
         cur = conn.cursor()
 
         # create the tiles table
@@ -330,6 +332,11 @@ def run(self, processes=4):
             "(zoom_level integer, tile_column integer, "
             "tile_row integer, tile_data blob);"
         )
+
+        # create index on tiles for efficient access to this table
+        cur.execute(
+            "CREATE UNIQUE  INDEX IF NOT EXISTS tile_index on tiles (zoom_level, tile_column, tile_row);")
+
         # create empty metadata
         cur.execute("CREATE TABLE metadata (name text, value text);")
 
@@ -371,6 +378,7 @@ def run(self, processes=4):
             constrained_bbox = list(mercantile.bounds(self.bounding_tile))
             tiles = _make_tiles(constrained_bbox, "EPSG:4326", self.min_z, self.max_z)
 
+        tilesCount = 0
         for tile, contents in self.pool.imap_unordered(self.run_function, tiles):
             x, y, z = tile
 
@@ -384,8 +392,13 @@ def run(self, processes=4):
                 "VALUES (?, ?, ?, ?);",
                 (z, x, tiley, buffer(contents)),
             )
+            tilesCount = tilesCount + 1
+            # commit data every 1000 tiles (about 150 Mo)
+            # Otherwise, the file .mbtiles-wal becomes huge (same size as the final file). The result is the need to have twice the size of the final Mbtiles on the hard drive
+            if (tilesCount % 1000 == 0):
+                conn.commit()
 
-            conn.commit()
+        conn.commit()
 
         conn.close()