diff --git a/init-server.py b/init-server.py index 2f34b7a..22e2fa8 100755 --- a/init-server.py +++ b/init-server.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import sys import click from dotenv import dotenv_values @@ -6,10 +7,10 @@ from ssh_lib.config import scripts from ssh_lib.kernel import set_cpu_governor, setup_kernel_settings -from ssh_lib.nginx import certbot, nginx +from ssh_lib.nginx import certbot, k6, nginx from ssh_lib.pkg_base import pkg_base, pkg_clean, pkg_upgrade from ssh_lib.planetiler import TILE_GEN_BIN, install_planetiler -from ssh_lib.utils import add_user, enable_sudo, put, setup_time, sudo_cmd +from ssh_lib.utils import add_user, enable_sudo, put, reboot, setup_time, sudo_cmd def prepare_shared(c): @@ -51,6 +52,7 @@ def prepare_tile_gen(c): def prepare_http_host(c): nginx(c) certbot(c) + k6(c) @click.command() @@ -59,14 +61,16 @@ def prepare_http_host(c): @click.option('--user', help='SSH user (if not in .ssh/config)') @click.option('--tile-gen', is_flag=True, help='Install tile-gen task') @click.option('--http-host', is_flag=True, help='Install http-host task') +@click.option('--reboot', 'do_reboot', is_flag=True, help='Reboot after installation') +@click.option('--debug', is_flag=True) @click.option( '--skip-shared', is_flag=True, help='Skip the shared installtion step (useful for development)' ) -def main(hostname, user, port, tile_gen, http_host, skip_shared): +def main(hostname, user, port, tile_gen, http_host, skip_shared, do_reboot, debug): if not click.confirm(f'Run script on {hostname}?'): return - if not tile_gen and not http_host: + if not tile_gen and not http_host and not debug: tile_gen = click.confirm('Would you like to install tile-gen task?') http_host = click.confirm('Would you like to install http-host task?') if not tile_gen and not http_host: @@ -89,6 +93,10 @@ def main(hostname, user, port, tile_gen, http_host, skip_shared): port=port, ) + if debug: + debug_tmp(c) + sys.exit() + if not skip_shared: prepare_shared(c) @@ -98,6 +106,13 @@ def main(hostname, user, port, tile_gen, http_host, skip_shared): if http_host: prepare_http_host(c) + if do_reboot: + reboot(c) + + +def debug_tmp(c): + k6(c) + if __name__ == '__main__': main() diff --git a/scripts/benchmark/.gitignore b/scripts/benchmark/.gitignore new file mode 100644 index 0000000..2211df6 --- /dev/null +++ b/scripts/benchmark/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/scripts/benchmark/create_path_list.py b/scripts/benchmark/create_path_list.py new file mode 100644 index 0000000..3d922fb --- /dev/null +++ b/scripts/benchmark/create_path_list.py @@ -0,0 +1,27 @@ +import json + + +with open('access.log') as fp: + json_lines = fp.readlines() + +paths = [] +for i, line in enumerate(json_lines): + log_data = json.loads(line) + if log_data['status'] != 200: + continue + + if log_data['request_method'] != 'GET': + continue + + uri = log_data['uri'] + + if 'tiles/' not in uri or not uri.endswith('.pbf'): + continue + + path = log_data['uri'].split('tiles/')[1] + paths.append(path + '\n') + + print(f'{i / len(json_lines) * 100:.1f}%') + +with open('path_list.txt', 'w') as fp: + fp.writelines(paths) diff --git a/ssh_lib/nginx.py b/ssh_lib/nginx.py index febb22f..b7589b1 100644 --- a/ssh_lib/nginx.py +++ b/ssh_lib/nginx.py @@ -15,14 +15,16 @@ def nginx(c): codename = ubuntu_codename(c) if not exists(c, '/usr/sbin/nginx'): - put_str( + sudo_cmd( c, - '/etc/apt/sources.list.d/nginx.list', - f'deb http://nginx.org/packages/mainline/ubuntu {codename} nginx', + 'curl https://nginx.org/keys/nginx_signing.key ' + '| gpg --dearmor ' + '| sudo tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null', ) - sudo_cmd( + put_str( c, - 'wget --quiet -O - http://nginx.org/keys/nginx_signing.key | apt-key add -', + '/etc/apt/sources.list.d/nginx.list', + f'deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/mainline/ubuntu {codename} nginx', ) apt_get_update(c) apt_get_install(c, 'nginx') @@ -40,7 +42,8 @@ def nginx(c): c.sudo( 'openssl req -x509 -nodes -days 365 -newkey rsa:2048 ' '-keyout /etc/nginx/ssl/dummy.key -out /etc/nginx/ssl/dummy.crt ' - '-subj "/C=US/ST=Dummy/L=Dummy/O=Dummy/CN=example.com"' + '-subj "/C=US/ST=Dummy/L=Dummy/O=Dummy/CN=example.com"', + hide=True, ) put(c, f'{config}/nginx/nginx.conf', '/etc/nginx/') @@ -61,3 +64,19 @@ def certbot(c): apt_get_purge(c, 'certbot') c.sudo('snap install --classic certbot', warn=True) + + +def k6(c): + sudo_cmd( + c, + 'curl https://dl.k6.io/key.gpg ' + '| gpg --dearmor ' + '| tee /usr/share/keyrings/k6-archive-keyring.gpg >/dev/null', + ) + put_str( + c, + '/etc/apt/sources.list.d/k6.list', + 'deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main', + ) + apt_get_update(c) + apt_get_install(c, 'k6') diff --git a/ssh_lib/pkg_base.py b/ssh_lib/pkg_base.py index 37a73dd..851fdc3 100644 --- a/ssh_lib/pkg_base.py +++ b/ssh_lib/pkg_base.py @@ -36,13 +36,17 @@ def pkg_clean(c): def pkg_base(c): pkg_list = [ + 'lsb-release', 'wget', - 'gpg', 'git', + # + 'gnupg2', 'gnupg-agent', + 'ca-certificates', + 'ubuntu-keyring', + # 'nload', 'iftop', - 'snapd', # 'python3', 'python3-venv',