diff --git a/scripts/extract_mbtiles/extract_mbtiles.py b/scripts/extract_mbtiles/extract_mbtiles.py index 05e0363..8b5f13b 100755 --- a/scripts/extract_mbtiles/extract_mbtiles.py +++ b/scripts/extract_mbtiles/extract_mbtiles.py @@ -43,6 +43,8 @@ def cli(mbtiles_path: Path, dir_path: Path): if 'planet' in mbtiles_path.parent.name: assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14) + print('DONE') + def write_metadata(c, *, dir_path): metadata = dict(c.execute('select name, value from metadata').fetchall()) @@ -72,9 +74,6 @@ def write_tile_files(c, *, dir_path): c.execute('select zoom_level, tile_column, tile_row, tile_data_id from tiles_shallow') for i, row in enumerate(c, start=1): - if i < 4678400: - continue - z = row[0] x = row[1] y = flip_y(z, row[2]) diff --git a/scripts/shrink_btrfs/shrink_btrfs.py b/scripts/shrink_btrfs/shrink_btrfs.py index 74b3dd1..bda83ca 100755 --- a/scripts/shrink_btrfs/shrink_btrfs.py +++ b/scripts/shrink_btrfs/shrink_btrfs.py @@ -40,25 +40,28 @@ def cli(btrfs_img: Path): mnt_dir = Path(tempfile.mkdtemp(dir=current_dir, prefix='tmp_shrink_')) subprocess.run(['mount', '-t', 'btrfs', btrfs_img, mnt_dir], check=True) - # needs to start with a balancing - # https://btrfs.readthedocs.io/en/latest/Balance.html - # https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html - print('Starting btrfs balancing') - p = subprocess.run( - ['btrfs', 'balance', 'start', '-dusage=100', mnt_dir], capture_output=True, text=True - ) - if p.returncode: - # subprocess.run(['umount', mnt_dir]) - # mnt_dir.rmdir() - print(f'Balance error: {p.stdout} {p.stderr}') - print('Balancing done') + # shink until max. 10 MB left or reached SMALLEST_SIZE or failure + while True: + # needs to start with a balancing + # https://btrfs.readthedocs.io/en/latest/Balance.html + # https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html + do_balancing(mnt_dir) - # shink until max. 10 MB left, or failure - free_bytes = get_usage(mnt_dir, 'Device unallocated') - while free_bytes > 10_000_000: - if not shrink(mnt_dir, int(free_bytes * 0.9)): - break free_bytes = get_usage(mnt_dir, 'Device unallocated') + device_size = get_usage(mnt_dir, 'Device size') + shrink_idea = free_bytes * 0.9 + + # workaround for the SMALLEST_SIZE limit + if device_size - free_bytes < SMALLEST_SIZE: + shrink_idea = (device_size - SMALLEST_SIZE) * 0.9 + + # stop if 10 MB left + if shrink_idea < 10_000_000: + break + + # stop if process error + if not do_shrink(mnt_dir, shrink_idea): + break total_size = get_usage(mnt_dir, 'Device size') @@ -80,11 +83,22 @@ def get_usage(mnt: Path, key: str): return free -def shrink(mnt: Path, delta_size: int): +def do_shrink(mnt: Path, delta_size: float): + delta_size = int(delta_size) print(f'Trying to shrink by {delta_size//1_000_000} MB') p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt]) return p.returncode == 0 +def do_balancing(mnt: Path): + print('Starting btrfs balancing') + p = subprocess.run( + ['btrfs', 'balance', 'start', '-dusage=100', mnt], capture_output=True, text=True + ) + if p.returncode: + print(f'Balance error: {p.stdout} {p.stderr}') + print('Balancing done') + + if __name__ == '__main__': cli() diff --git a/scripts/tile_gen/extract_btrfs.sh b/scripts/tile_gen/extract_btrfs.sh index 91321d6..128aaa5 100644 --- a/scripts/tile_gen/extract_btrfs.sh +++ b/scripts/tile_gen/extract_btrfs.sh @@ -3,8 +3,8 @@ set -e sudo umount mnt_rw || true sudo umount mnt_rw2 || true -rm -rf mnt_rw* -rm -f image*.btrfs +rm -rf mnt_rw* tmp_* +rm -f *.btrfs *.gz rm -f *.log # make an empty file that's definitely bigger then the current OSM output @@ -33,16 +33,42 @@ sudo mount -v \ sudo mount -v \ -t btrfs \ -o noacl,nobarrier,noatime,max_inline=4096 \ - image.btrfs mnt_rw2 + image2.btrfs mnt_rw2 sudo chown ofm:ofm -R mnt_rw mnt_rw2 ../../tile_gen/venv/bin/python ../../tile_gen/extract_mbtiles.py output.mbtiles mnt_rw/extract \ > extract_out.log 2> extract_err.log -# we need to extract, delete and rsync onto a new partition +grep fixed extract_out.log > dedupl_fixed.log || true + +# we need to extract, delete dedupl and rsync onto a new partition # otherwise the partition image stays big -rsync -aH mnt_rw/extract/ mnt_rw2/extract/ > rsync_out.log 2> rsync_err.log +rsync -avH mnt_rw/extract/ mnt_rw2/extract/ > rsync_out.log 2> rsync_err.log + + +# collect stats +{ +echo -e "df -h" +df -h mnt_rw2 + +echo -e "\n\nbtrfs filesystem df" +btrfs filesystem df mnt_rw2 + +echo -e "\n\nbtrfs filesystem du -s" +btrfs filesystem du -s mnt_rw2 + +echo -e "\n\nbtrfs filesystem show" +btrfs filesystem show mnt_rw2 + +echo -e "\n\nbtrfs filesystem usage" +btrfs filesystem usage mnt_rw2 + +echo -e "\n\ncompsize -x" +compsize -x mnt_rw2 +} > stats.txt + + sudo umount mnt_rw sudo umount mnt_rw2 @@ -52,9 +78,12 @@ sudo ../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image2.btrfs > shrink_out.log 2> shrink_err.log -#rm image.btrfs -#mv image2.btrfs done.btrfs -# pigz -k image.btrfs --fast +rm image.btrfs +mv image2.btrfs done.btrfs + +pigz done.btrfs --fast + +echo DONE