work

README.md

@@ -1 +1,4 @@
 # openfreemap
+
+# TODO
+not deleted dedupl files

config/limits/limits-ofm.conf

@@ -0,0 +1,4 @@
+* soft nofile 1048576
+* hard nofile 1048576
+root soft nofile 1048576
+root hard nofile 1048576

config/nginx/nginx.conf

@@ -59,10 +59,6 @@ http {
         '"http_referrer": "$http_referer", '
         '"http_x_forwarded_for": "$http_x_forwarded_for", '
         '"http_user_agent": "$http_user_agent", '
-        '"upstream_response_time": "$upstream_response_time", '
-       # '"upstream_connect_time": "$upstream_connect_time", '
-        '"upstream_header_time": "$upstream_header_time", '
-        '"upstream_cache_status": "$upstream_cache_status", '
         '"host": "$host", '
         '"uri": "$uri", '
         '"http_cf_connecting_ip": "$http_cf_connecting_ip", '

config/sysctl/60-optim.conf → config/sysctl/60-ofm.conf

@@ -1,4 +1,4 @@
 vm.swappiness = 1
 
 net.core.somaxconn = 65535
-
+fs.file-max = 100000

init-server.py

@@ -5,9 +5,10 @@
 from dotenv import dotenv_values
 from fabric import Config, Connection
 
-from ssh_lib.config import scripts
+from ssh_lib.benchmark import benchmark, c1000k, k6
+from ssh_lib.config import config, scripts
 from ssh_lib.kernel import set_cpu_governor, setup_kernel_settings
-from ssh_lib.nginx import certbot, k6, nginx
+from ssh_lib.nginx import certbot, nginx
 from ssh_lib.pkg_base import pkg_base, pkg_clean, pkg_upgrade
 from ssh_lib.planetiler import TILE_GEN_BIN, install_planetiler
 from ssh_lib.utils import add_user, enable_sudo, put, reboot, setup_time, sudo_cmd
@@ -52,7 +53,18 @@ def prepare_tile_gen(c):
 def prepare_http_host(c):
     nginx(c)
     certbot(c)
-    k6(c)
+    c1000k(c)
+
+
+def debug_tmp(c):
+    c.sudo('rm -rf /data/ofm/logs')
+    c.sudo('mkdir -p /data/ofm/logs')
+    put(c, f'{config}/nginx/nginx.conf', '/etc/nginx/')
+    put(c, f'{scripts}/http_host/nginx_site.conf', '/data/nginx/sites')
+    c.sudo('nginx -t')
+    c.sudo('service nginx restart')
+
+    benchmark(c)
 
 
 @click.command()
@@ -110,9 +122,5 @@ def main(hostname, user, port, tile_gen, http_host, skip_shared, do_reboot, debu
         reboot(c)
 
 
-def debug_tmp(c):
-    k6(c)
-
-
 if __name__ == '__main__':
     main()

kernel-ideas.txt

@@ -3,8 +3,19 @@ tcp_fin_timeout
 tcp_max_syn_backlog
 TCP max buffer size
 
-Increase File Descriptors Limit
-
 Disable Swapping
 
+fs.file-max # not needed, recent Ubuntu has it on high by default
+net.core.netdev_max_backlog = 262144
+net.core.rmem_max = 16777216
+net.core.somaxconn = 262144
+net.core.wmem_max = 16777216
+net.ipv4.tcp_max_syn_backlog = 262144
+net.ipv4.tcp_max_tw_buckets = 6000000
+net.ipv4.tcp_no_metrics_save = 1
+net.ipv4.tcp_rmem = 4096 87380 16777216
+net.ipv4.tcp_syn_retries = 2
+net.ipv4.tcp_synack_retries = 2
+net.ipv4.tcp_tw_reuse = 1
+net.ipv4.tcp_wmem = 4096 65536 16777216
 

scripts/benchmark/wrk_custom_list.lua

@@ -0,0 +1,29 @@
+local counter = 1
+local lines = {}
+local base_path = "/planet/20231208_091355/tiles/"
+local file_path = "/data/ofm/benchmark/path_list_small.txt"
+
+for line in io.lines(file_path) do
+    table.insert(lines, base_path .. line)
+end
+
+local function getNextUrl()
+    -- Get the next URL from the list
+    local url_path = lines[counter]
+    counter = counter + 1
+
+    -- If we've gone past the end of the list, wrap around to the start
+    if counter > #lines then
+        counter = 1
+    end
+
+    return url_path
+end
+
+request = function()
+    -- Return the request object with the current URL path
+    local path = getNextUrl()
+    local headers = {}
+    headers["Host"] = "ofm"
+    return wrk.format('GET', path, headers, nil)
+end

scripts/http_host/mount_btrfs.sh

@@ -0,0 +1,5 @@
+mkdir -p mnt_ro
+sudo mount -v \
+  -t btrfs \
+  -o ro \
+  image.btrfs mnt_ro

scripts/http_host/nginx_site.conf

@@ -0,0 +1,17 @@
+server {
+    server_name ofm tiles.openfreemaps.org;
+    # test with
+    # curl -H "Host: ofm" http://localhost/planet/20231208_091355/tiles/7/72/48.pbf
+
+
+    #access_log /data/ofm/logs/nginx-access.log access_json;
+    access_log off;
+    error_log /data/ofm/logs/nginx-error.log;
+
+    location /planet/20231208_091355 {
+        gzip off;
+
+        alias /data/ofm/runs/planet_20231208_091355/mnt_ro/extract;
+        autoindex on;  # Enables listing of directory
+    }
+}

scripts/tile_gen/extract_btrfs.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-sudo umount mnt || true
-rm -rf mnt
+sudo umount mnt_rw || true
+rm -rf mnt_rw
 rm -f image.btrfs
 
 
@@ -16,21 +16,21 @@ mkfs.btrfs -v \
 
 # https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
 # compression doesn't make sense, data is already gzip compressed
-mkdir -p mnt
+mkdir -p mnt_rw
 sudo mount -v \
   -t btrfs \
   -o noacl,nobarrier,noatime,max_inline=4096 \
-  image.btrfs mnt
+  image.btrfs mnt_rw
 
 
-sudo chown ofm:ofm -R mnt
+sudo chown ofm:ofm -R mnt_rw
 
-../../tile_gen/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt/extract \
+../../tile_gen/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt_rw/extract \
   > "extract_out.log" 2> "extract_err.log"
 
-sudo umount mnt
+sudo umount mnt_rw
 
 ../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image.btrfs
 
-
+# pigz -k image.btrfs --fast
 

ssh_lib/benchmark.py

@@ -0,0 +1,36 @@
+from ssh_lib.config import scripts
+from ssh_lib.utils import apt_get_install, apt_get_update, put, put_str, sudo_cmd
+
+
+def k6(c):
+    sudo_cmd(
+        c,
+        'curl https://dl.k6.io/key.gpg '
+        '| gpg --dearmor '
+        '| tee /usr/share/keyrings/k6-archive-keyring.gpg >/dev/null',
+    )
+    put_str(
+        c,
+        '/e' 'tc/apt/sources.list.d/k6.list',
+        'deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main',
+    )
+    apt_get_update(c)
+    apt_get_install(c, 'k6')
+
+
+def c1000k(c):
+    c.run('wget https://github.com/ideawu/c1000k/archive/master.zip -O tmp.zip')
+    c.run('unzip -o tmp.zip')
+    c.run('rm tmp.zip')
+    c.run('cd c1000k-master && make')
+
+    # usage
+    # ./server 7000
+    # ./client 127.0.0.1 7000
+    # make sure it runs till 1 million
+
+
+def benchmark(c):
+    apt_get_install(c, 'wrk')
+    c.sudo('mkdir -p /data/ofm/benchmark')
+    put(c, f'{scripts}/benchmark/wrk_custom_list.lua', '/data/ofm/benchmark')

ssh_lib/kernel.py

@@ -3,7 +3,8 @@
 
 
 def setup_kernel_settings(c):
-    put(c, f'{config}/sysctl/60-optim.conf', '/etc/sysctl.d/')
+    put(c, f'{config}/sysctl/60-ofm.conf', '/etc/sysctl.d/')
+    put(c, f'{config}/limits/limits-ofm.conf', '/etc/security/limits.d/')
 
 
 def set_cpu_governor(c):

ssh_lib/nginx.py

@@ -64,19 +64,3 @@ def certbot(c):
 
     apt_get_purge(c, 'certbot')
     c.sudo('snap install --classic certbot', warn=True)
-
-
-def k6(c):
-    sudo_cmd(
-        c,
-        'curl https://dl.k6.io/key.gpg '
-        '| gpg --dearmor '
-        '| tee /usr/share/keyrings/k6-archive-keyring.gpg >/dev/null',
-    )
-    put_str(
-        c,
-        '/etc/apt/sources.list.d/k6.list',
-        'deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main',
-    )
-    apt_get_update(c)
-    apt_get_install(c, 'k6')

ssh_lib/pkg_base.py

@@ -39,6 +39,8 @@ def pkg_base(c):
         'lsb-release',
         'wget',
         'git',
+        'build-essential',
+        'unzip',
         #
         'gnupg2',
         'gnupg-agent',