From 70a3f0068d2d39740b588bdca0414921035d29d3 Mon Sep 17 00:00:00 2001 From: esgn <5435148+esgn@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:46:04 +0100 Subject: [PATCH] refactoring in progress --- pgtune.sh | 742 +++++++++++------- .../10_linux_web_8GB_8_1000_hdd.txt | 2 +- .../10_linux_web_8GB_8_1000_san.txt | 2 +- .../10_linux_web_8GB_8_1000_ssd.txt | 2 +- .../11_linux_web_8GB_8_1000_hdd.txt | 2 +- .../11_linux_web_8GB_8_1000_san.txt | 2 +- .../11_linux_web_8GB_8_1000_ssd.txt | 2 +- .../12_linux_web_8GB_8_1000_hdd.txt | 2 +- .../12_linux_web_8GB_8_1000_san.txt | 2 +- .../12_linux_web_8GB_8_1000_ssd.txt | 2 +- .../13_linux_web_8GB_8_1000_hdd.txt | 2 +- .../13_linux_web_8GB_8_1000_san.txt | 2 +- .../13_linux_web_8GB_8_1000_ssd.txt | 2 +- .../14_linux_web_8GB_8_1000_hdd.txt | 2 +- .../14_linux_web_8GB_8_1000_san.txt | 2 +- .../14_linux_web_8GB_8_1000_ssd.txt | 2 +- .../15_linux_web_8GB_8_1000_hdd.txt | 2 +- .../15_linux_web_8GB_8_1000_san.txt | 2 +- .../15_linux_web_8GB_8_1000_ssd.txt | 2 +- test/scripts/pull_testing_images.sh | 26 +- test/scripts/test_pgtune.sh | 2 +- 21 files changed, 487 insertions(+), 319 deletions(-) diff --git a/pgtune.sh b/pgtune.sh index b656f50..e9fa6c8 100644 --- a/pgtune.sh +++ b/pgtune.sh @@ -1,10 +1,22 @@ #!/usr/bin/env bash +####################################### +# Constants definition +####################################### KB=1024 MB=1048576 GB=1073741824 +TOTAL_MEM=0 +CPU_NUM=0 +STORAGE_TYPE="" +CONN_NUM=0 +DB_TYPE="" +DB_VERSION=0 -show_help() { +####################################### +# Display help message +####################################### +function show_help() { cat << EOF Usage: ${0##*/} [-h] [-v PG_VERSION] [-t DB_TYPE] [-m TOTAL_MEM] [-u CPU_COUNT] [-c MAX_CONN] [-s STGE_TYPE] @@ -25,53 +37,96 @@ It produces a postgresql.conf file based on supplied parameters. accepted values: integer between 1 and 9999 CPUs = threads per core * cores per socket * sockets default value: this script will try to determine the CPUs count and exit in case of failure - -c MAX_CONN (optional) Maximum number of PostgreSQL client connections + -c MAX_CONN (optional) Maxsimum number of PostgreSQL client connections accepted values: integer between 20 and 9999 - default value: preset corresponding to db_type + default value: preset corresponding to DB_TYPE -s STGE_TYPE (optional) Type of data storage device used with PostgreSQL accepted values: hdd, ssd, san - default value: this script will try to determine the storage type (san not supported) and use hdd - value in case of failure. + default value: this script will try to determine the storage type (san not supported) + and use hdd value in case of failure. EOF } -_warn() { - echo >&2 "[pgtune.sh] $*" +####################################### +# Utility function to print a warning. +# Arguments: +# Message to print as warning +####################################### +function _warn() { + echo "[pgtune.sh] warning: $*" >&2 } -_input_error() { - echo >&2 "[pgtune.sh] input error: $*" +####################################### +# Utility function to print an error. +# related to an input value +# Arguments: +# Message to print as input error +####################################### +function _input_error() { + echo "[pgtune.sh] input error: $*" >&2 exit 1 } -_error() { - echo >&2 "[pgtune.sh] error: $*" +####################################### +# Utility functions to print error messages. +# Arguments: +# Message to print as error +####################################### +function _error() { + echo "[pgtune.sh] error: $*" >&2 exit 2 } -get_total_ram () { - local total_ram=$(cat /proc/meminfo | grep -i 'memtotal' | grep -o '[[:digit:]]*') - if [[ -z $total_ram ]] || [[ "$total_ram" -eq "0" ]] - then - _error "cannot detect total memory size, terminating script. Please supply -m TOTAL_MEM." +####################################### +# Get total RAM amount in kB from /proc/meminfo. +# Arguments: +# None +# Outputs: +# Total RAM in kB +####################################### +function get_total_ram () { + local -i total_ram + total_ram=$(< /proc/meminfo grep -i 'memtotal' | grep -o '[[:digit:]]*') + if [[ -z $total_ram ]] || [[ "$total_ram" -eq "0" ]]; then + _error "Cannot detect total memory size, terminating script. Please supply -m TOTAL_MEM." fi echo $total_ram } -get_cpu_count () { - local cpu_count=$(nproc --all) - if [[ -z $cpu_count ]] || [[ "$cpu_count" -eq "0" ]] - then - _error "cannot detect cpu count, terminating script. Please supply -u CPU_COUNT." +####################################### +# Get CPU cores count using nproc command. +# Arguments: +# None +# Outputs: +# Number of CPU cores +####################################### +function get_cpu_count () { + local -i cpu_count + cpu_count=$(nproc --all) + if [[ -z $cpu_count ]] || [[ "$cpu_count" -eq "0" ]]; then + _error "Cannot detect CPU count, terminating script. Please supply -u CPU_COUNT." fi echo $cpu_count } -get_disk_type () { +####################################### +# Try and guess the disk type using findmnt command +# and information available in /sys/block. +# Could greatly be improved. +# Globals: +# PGDATA: used +# Arguments: +# None +# Outputs: +# Type of disks : 'ssd' or 'hdd' +####################################### +function get_disk_type () { # PGDATA should always be defined in base postgres image # findmnt should also be installed by default - local disk_name=$(basename $(findmnt -v -n -o SOURCE --target $PGDATA 2>/dev/null) 2>/dev/null) - local disk_type_clue=$(cat /sys/block/$disk_name/queue/rotational 2>/dev/null) + local disk_name + local disk_type_clue + disk_name="$(basename "$(findmnt -v -n -o SOURCE --target "$PGDATA" 2>/dev/null)" 2>/dev/null)" + disk_type_clue="$(cat /sys/block/"$disk_name"/queue/rotational 2>/dev/null)" case "$disk_type_clue" in "0") disk_type="ssd" @@ -80,15 +135,24 @@ get_disk_type () { disk_type="hdd" ;; *) - _warn "cannot detect disk type, hdd type will be used. Supply -s STGE_TYPE if necessary." + _warn "Cannot detect disk type. 'hdd' type will be used as STGE_TYPE argument was not supplied." disk_type="hdd" ;; esac echo $disk_type } -set_db_default_values() { - case "$db_version" in +####################################### +# Set default config values depending +# on PostgreSQL version number. +# Globals: +# DB_VERSION: used +# max_worker_processes: modified +# max_parallel_workers_per_gather: modified +# max_parallel_workers: modified +####################################### +function set_db_default_values() { + case "$DB_VERSION" in "9.5") max_worker_processes=8 ;; @@ -107,84 +171,113 @@ set_db_default_values() { esac } -set_shared_buffers() { - case "$db_type" in +####################################### +# Set shared_buffers config value. +# Globals: +# DB_TYPE: used +# TOTAL_MEM: used +# shared_buffers: modified +####################################### +function set_shared_buffers() { + case "$DB_TYPE" in "web") - shared_buffers=$(( $total_mem/4 )) + shared_buffers=$(( TOTAL_MEM/4 )) ;; "oltp") - shared_buffers=$(( $total_mem/4 )) + shared_buffers=$(( TOTAL_MEM/4 )) ;; "dw") - shared_buffers=$(( $total_mem/4 )) + shared_buffers=$(( TOTAL_MEM/4 )) ;; "desktop") - shared_buffers=$(( $total_mem/16 )) + shared_buffers=$(( TOTAL_MEM/16 )) ;; "mixed") - shared_buffers=$(( $total_mem/4 )) + shared_buffers=$(( TOTAL_MEM/4 )) ;; *) - _error "unknown db_type, cannot calculate shared_buffers" + _error "unknown DB_TYPE, cannot calculate shared_buffers" ;; esac } -set_effective_cache_size() { - case "$db_type" in +####################################### +# Set set_effective_cache_size config value. +# Globals: +# DB_TYPE: used +# TOTAL_MEM: used +# effective_cache_size: modified +####################################### +function set_effective_cache_size() { + case "$DB_TYPE" in "web") - effective_cache_size=$(( $total_mem*3/4 )) + effective_cache_size=$(( TOTAL_MEM*3/4 )) ;; "oltp") - effective_cache_size=$(( $total_mem*3/4 )) + effective_cache_size=$(( TOTAL_MEM*3/4 )) ;; "dw") - effective_cache_size=$(( $total_mem*3/4 )) + effective_cache_size=$(( TOTAL_MEM*3/4 )) ;; "desktop") - effective_cache_size=$(( $total_mem/4 )) + effective_cache_size=$(( TOTAL_MEM/4 )) ;; "mixed") - effective_cache_size=$(( $total_mem*3/4 )) + effective_cache_size=$(( TOTAL_MEM*3/4 )) ;; *) - _error "unknown db_type, cannot calculate effective_cache_size" + _error "unknown DB_TYPE, cannot calculate effective_cache_size" ;; esac } -set_maintenance_work_mem() { - case "$db_type" in +####################################### +# Set set_maintenance_work_mem config value. +# Globals: +# DB_TYPE: used +# TOTAL_MEM: used +# effective_cache_size: modified +####################################### +function set_maintenance_work_mem() { + local mem_limit=$(( 2 * GB / KB )) + case "$DB_TYPE" in "web") - maintenance_work_mem=$(( $total_mem/16 )) + maintenance_work_mem=$(( TOTAL_MEM/16 )) ;; "oltp") - maintenance_work_mem=$(( $total_mem/16 )) + maintenance_work_mem=$(( TOTAL_MEM/16 )) ;; "dw") - maintenance_work_mem=$(( $total_mem/8 )) + maintenance_work_mem=$(( TOTAL_MEM/8 )) ;; "desktop") - maintenance_work_mem=$(( $total_mem/16 )) + maintenance_work_mem=$(( TOTAL_MEM/16 )) ;; "mixed") - maintenance_work_mem=$(( $total_mem/16 )) + maintenance_work_mem=$(( TOTAL_MEM/16 )) ;; *) - _error "unknown db_type, cannot calculate maintenance_work_mem" + _error "unknown DB_TYPE, cannot calculate maintenance_work_mem" ;; esac - local mem_limit=$(( 2 * $GB / $KB )) - if [ "$maintenance_work_mem" -gt "$mem_limit" ] - then + if [ "$maintenance_work_mem" -gt "$mem_limit" ]; then maintenance_work_mem=$mem_limit fi } -set_checkpoint_segments() { - if [ ${db_version%.*} -le 9 ] && [ ${db_version//./} -lt 95 ] - then - case "$db_type" in +####################################### +# Set checkpoint_segments, min_wal_size +# and max_wal_size config values. +# Globals: +# DB_VERSION: used +# DB_TYPE: used +# checkpoint_segments: modified +# min_wal_size: modified +# max_wal_size: modified +####################################### +function set_checkpoint_segments() { + if [ "${DB_VERSION%.*}" -le 9 ] && [ "${DB_VERSION//./}" -lt 95 ]; then + case "$DB_TYPE" in "web") checkpoint_segments=32 ;; @@ -201,69 +294,83 @@ set_checkpoint_segments() { checkpoint_segments=32 ;; *) - _error "unknown db_type, cannot calculate checkpoint_segments" + _error "unknown DB_TYPE, cannot calculate checkpoint_segments" ;; esac else - case "$db_type" in + case "$DB_TYPE" in "web") - min_wal_size=$(( 1024 * $MB / $KB )) - max_wal_size=$(( 4096 * $MB / $KB )) + min_wal_size=$(( 1024 * MB / KB )) + max_wal_size=$(( 4096 * MB / KB )) ;; "oltp") - min_wal_size=$(( 2048 * $MB / $KB )) - max_wal_size=$(( 8192 * $MB / $KB )) + min_wal_size=$(( 2048 * MB / KB )) + max_wal_size=$(( 8192 * MB / KB )) ;; "dw") - min_wal_size=$(( 4096 * $MB / $KB )) - max_wal_size=$(( 16384 * $MB / $KB )) + min_wal_size=$(( 4096 * MB / KB )) + max_wal_size=$(( 16384 * MB / KB )) ;; "desktop") - min_wal_size=$(( 100 * $MB / $KB )) - max_wal_size=$(( 2048 * $MB / $KB )) + min_wal_size=$(( 100 * MB / KB )) + max_wal_size=$(( 2048 * MB / KB )) ;; "mixed") - min_wal_size=$(( 1024 * $MB / $KB )) - max_wal_size=$(( 4096 * $MB / $KB )) + min_wal_size=$(( 1024 * MB / KB )) + max_wal_size=$(( 4096 * MB / KB )) ;; *) - _error "unknown db_type, cannot calculate min_wal_size" + _error "unknown DB_TYPE, cannot calculate min_wal_size" ;; esac fi } -set_checkpoint_completion_target() { +####################################### +# Set set_checkpoint_completion_target config value. +# Globals: +# checkpoint_completion_target: modified +####################################### +function set_checkpoint_completion_target() { # based on https://github.com/postgres/postgres/commit/bbcc4eb2 checkpoint_completion_target=0.9 } -set_wal_buffers() { +####################################### +# Set set_wal_buffers config value. +# Globals: +# shared_buffers: used +# wal_buffers: modified +####################################### +function set_wal_buffers() { # Follow auto-tuning guideline for wal_buffers added in 9.1, where it's # set to 3% of shared_buffers up to a maximum of 16MB. - wal_buffers=$(( 3 * $shared_buffers / 100 )) - local max_wal_buffers=$(( 16 * $MB / $KB )) - if [ "$wal_buffers" -gt "$max_wal_buffers" ] - then + wal_buffers=$(( 3 * shared_buffers / 100 )) + local max_wal_buffers=$(( 16 * MB / KB )) + if [ "$wal_buffers" -gt "$max_wal_buffers" ];then wal_buffers=$max_wal_buffers fi # It's nice if wal_buffers is an even 16MB if it's near that number. Since # that is a common case on Windows, where shared_buffers is clipped to 512MB, # round upwards in that situation local near_max_wal_buffers=$(( 14 * MB / KB )) - if [ "$wal_buffers" -gt "$near_max_wal_buffers" ] && [ "$wal_buffers" -lt "$max_wal_buffers" ] - then + if [ "$wal_buffers" -gt "$near_max_wal_buffers" ] && [ "$wal_buffers" -lt "$max_wal_buffers" ]; then wal_buffers=$max_wal_buffers fi # if less, than 32 kb, than set it to minimum - if [ "$wal_buffers" -lt 32 ] - then + if [ "$wal_buffers" -lt 32 ]; then wal_buffers=32 fi } -set_default_statistics_target() { - case "$db_type" in +####################################### +# Set set_default_statistics_target config value. +# Globals: +# DB_TYPE: used +# default_statistics_target: modified +####################################### +function set_default_statistics_target() { + case "$DB_TYPE" in "web") default_statistics_target=100 ;; @@ -280,13 +387,19 @@ set_default_statistics_target() { default_statistics_target=100 ;; *) - _error "unknown db_type, cannot calculate default_statistics_target" + _error "unknown DB_TYPE, cannot calculate default_statistics_target" ;; esac } -set_random_page_cost() { - case "$storage_type" in +####################################### +# Set set_random_page_cost config value. +# Globals: +# STORAGE_TYPE: used +# random_page_cost: modified +####################################### +function set_random_page_cost() { + case "$STORAGE_TYPE" in "ssd") random_page_cost=1.1 ;; @@ -297,13 +410,19 @@ set_random_page_cost() { random_page_cost=1.1 ;; *) - _error "unknown storage_type, cannot calculate random_page_cost" + _error "unknown STORAGE_TYPE, cannot calculate random_page_cost" ;; esac } -set_effective_io_concurrency() { - case "$storage_type" in +####################################### +# Set set_random_page_cost config value. +# Globals: +# STORAGE_TYPE: used +# effective_io_concurrency: modified +####################################### +function set_effective_io_concurrency() { + case "$STORAGE_TYPE" in "ssd") effective_io_concurrency=200 ;; @@ -314,56 +433,74 @@ set_effective_io_concurrency() { effective_io_concurrency=300 ;; *) - _error "unknown storage_type, cannot calculate effective_io_concurrency" + _error "unknown STORAGE_TYPE, cannot calculate effective_io_concurrency" ;; esac } -set_parallel_settings() { +####################################### +# Set max_worker_processes, max_parallel_workers_per_gather, +# workers_per_gather, maintenance_workers +# and max_parallel_maintenance_workers config values. +# These values are stored in a parallel_settings array. +# Globals: +# CPU_NUM: used +# DB_VERSION: used +# DB_TYPE: used +# max_worker_processes: modified +# max_parallel_workers_per_gather: modified +# workers_per_gather: modified +# maintenance_workers: modified +# max_parallel_maintenance_workers: modified +# parallel_settings: modified +####################################### +function set_parallel_settings() { declare -Ag parallel_settings - if [ "$cpu_num" -lt 2 ] || ( [ ${db_version%.*} -le 9 ] && [ ${db_version//./} -lt 95 ] ) - then + if [ "$CPU_NUM" -lt 2 ] || ( [ "${DB_VERSION%.*}" -le 9 ] && [ "${DB_VERSION//./}" -lt 95 ] ); then return 0 fi - parallel_settings[max_worker_processes]="$cpu_num" - if [ "${db_version//./}" -ge "96" ] || [ ${db_version%.*} -ge 10 ] - then - workers_per_gather=$(( $cpu_num / 2 )) - if [ $workers_per_gather -gt 4 ] && [[ "$db_type" != "dw" ]] - then + parallel_settings[max_worker_processes]="$CPU_NUM" + if [ "${DB_VERSION//./}" -ge "96" ] || [ "${DB_VERSION%.*}" -ge 10 ]; then + workers_per_gather=$(( CPU_NUM / 2 )) + if [ $workers_per_gather -gt 4 ] && [[ "$DB_TYPE" != "dw" ]]; then workers_per_gather=4 fi parallel_settings[max_parallel_workers_per_gather]="$workers_per_gather" fi - if [ ${db_version%.*} -ge 10 ] - then - parallel_settings[max_parallel_workers]="$cpu_num" + if [ "${DB_VERSION%.*}" -ge 10 ]; then + parallel_settings[max_parallel_workers]="$CPU_NUM" fi - if [ ${db_version%.*} -ge 11 ] - then - maintenance_workers=$(( $cpu_num / 2 )) - if [ $maintenance_workers -gt 4 ] - then + if [ "${DB_VERSION%.*}" -ge 11 ]; then + maintenance_workers=$(( CPU_NUM / 2 )) + if [ $maintenance_workers -gt 4 ]; then maintenance_workers=4 fi parallel_settings[max_parallel_maintenance_workers]="$maintenance_workers" fi } -set_work_mem() { - parallel_for_work_mem=1 - if [ "${#parallel_settings[@]}" -gt "0" ] - then - if [[ ${parallel_settings[max_parallel_workers_per_gather]} ]] - then +####################################### +# Set work_mem config value. +# Globals: +# DB_TYPE: used +# parallel_settings: used +# max_parallel_workers_per_gather: used +# TOTAL_MEM: used +# shared_buffers: used +# CONN_NUM: used +# work_mem: modified +####################################### +function set_work_mem() { + local -i parallel_for_work_mem=1 + if [ "${#parallel_settings[@]}" -gt "0" ]; then + if [[ ${parallel_settings[max_parallel_workers_per_gather]} ]]; then parallel_for_work_mem=${parallel_settings[max_parallel_workers_per_gather]} fi - elif [ ! -z ${max_parallel_workers_per_gather+x} ] - then + elif [ ! -z ${max_parallel_workers_per_gather+x} ]; then parallel_for_work_mem=$max_parallel_workers_per_gather fi - work_mem_value=$(( ($total_mem - $shared_buffers) / ($conn_nb * 3) / $parallel_for_work_mem )) - case "$db_type" in + work_mem_value=$(( ($TOTAL_MEM - $shared_buffers) / ($CONN_NUM * 3) / $parallel_for_work_mem )) + case "$DB_TYPE" in "web") work_mem=$work_mem_value ;; @@ -380,29 +517,34 @@ set_work_mem() { work_mem=$(( $work_mem_value/2 )) ;; *) - _error "unknown db_type, cannot calculate work_mem" + _error "unknown DB_TYPE, cannot calculate work_mem" ;; esac - if [ $work_mem -lt 64 ] - then + if [ $work_mem -lt 64 ]; then work_mem=64 fi } -format_value() { +####################################### +# Format config option value +# Arguments: +# Config value to be formatted +# Number of spaces between value and unit +# Outputs: +# Formated value +####################################### +function format_value() { + # Set space value to 0 if the argument is missing with_space=${2:-0} - if [[ $with_space -eq "1" ]] - then + if [[ $with_space -eq "1" ]]; then space=" " else space="" fi - if [ $(( $1 % $MB )) -eq 0 ] - then + if [ $(( $1 % $MB )) -eq 0 ]; then formatted_value=$(( $1 / $MB ))$space"GB" echo $formatted_value - elif [ $(( $1 % $KB )) -eq 0 ] - then + elif [ $(( $1 % $KB )) -eq 0 ]; then formatted_value=$(( $1 / $KB ))$space"MB" echo $formatted_value else @@ -411,167 +553,189 @@ format_value() { fi } -total_mem=$(get_total_ram) || exit $? -cpu_num=$(get_cpu_count) || exit $? -storage_type=$(get_disk_type) -conn_nb=0 -db_type="web" -db_version=16 - -while getopts "hv:t:m:u:c:s:" opt; do - case $opt in - h) - show_help - exit 0 - ;; - v) - v=$OPTARG - if [ $v != "9.5" ] && \ - [ $v != "9.6" ] && \ - [ $v != "10" ] && \ - [ $v != "11" ] && \ - [ $v != "12" ] && \ - [ $v != "13" ] && \ - [ $v != "14" ] && \ - [ $v != "15" ] && \ - [ $v != "16" ] - then - _input_error "$v is not a valid PostgreSQL version number" - fi - db_version=$v - ;; - t) - t=$OPTARG - if [ $t != "web" ] && \ - [ $t != "oltp" ] && \ - [ $t != "dw" ] && \ - [ $t != "desktop" ] && \ - [ $t != "mixed" ] - then - _input_error "$t is not a valid database type identifier" - fi - db_type="$t" - ;; - m) - m=$OPTARG - if [[ $m == *"MB"* ]] - then - ram=${m%"MB"} - if [ "$ram" -lt "512" ] || [ "$ram" -gt "9999" ] - then - _input_error "total memory in MB must be greater than or equal to 512MB and less than or equal to 9999MB" +####################################### +# Main method +####################################### +function main() { + + while getopts "hv:t:m:u:c:s:" opt; do + case $opt in + h) + show_help + exit 0 + ;; + v) + v=$OPTARG + if [ $v != "9.5" ] && \ + [ $v != "9.6" ] && \ + [ $v != "10" ] && \ + [ $v != "11" ] && \ + [ $v != "12" ] && \ + [ $v != "13" ] && \ + [ $v != "14" ] && \ + [ $v != "15" ] && \ + [ $v != "16" ]; then + _input_error "$v is not a valid PostgreSQL version number" fi - ram=$(( $ram*$KB )) - elif [[ $m == *"GB"* ]] - then - ram=${m%"GB"} - if [ "$ram" -lt "1" ] || [ "$ram" -gt "9999" ] - then - _input_error "total memory in GB must be greater than or equal to 1GB and less than or equal to 9999GB" + DB_VERSION=$v + ;; + t) + t=$OPTARG + if [ $t != "web" ] && \ + [ $t != "oltp" ] && \ + [ $t != "dw" ] && \ + [ $t != "desktop" ] && \ + [ $t != "mixed" ]; then + _input_error "$t is not a valid database type identifier" fi - ram=$(( $ram*$MB )) - else - _input_error "$m does not contain a valid unit identifier (use MB or GB)" - fi - total_mem="$ram" - ;; - u) - u=$OPTARG - if [ "$u" -lt "1" ] || [ "$u" -gt "9999" ] - then - _input_error "CPU count must be greater than or equal to 1 and less than or equal to 9999" - fi - cpu_num=$u - ;; - c) - c=$OPTARG - if [ "$c" -lt "20" ] || [ "$c" -gt "9999" ] - then - _input_error "connections number be greater than or equal to 1 and less than or equal to 9999" - fi - conn_nb=$c - ;; - s) - s=$OPTARG - if [ $s != "hdd" ] && \ - [ $s != "ssd" ] && \ - [ $s != "san" ] - then - _input_error "$s is not a valid storage type identifier" - fi - storage_type="$s" - ;; - *) - show_help >&2 - exit 2 - ;; - esac -done - -if [ $conn_nb -eq "0" ] -then - case $db_type in - "web") - conn_nb=200 - ;; - "oltp") - conn_nb=300 - ;; - "dw") - conn_nb=40 - ;; - "desktop") - conn_nb=20 - ;; - "mixed") - conn_nb=100 - ;; - *) - conn_nb=20 - ;; - esac -fi - -set_db_default_values || exit $? -set_shared_buffers || exit $? -set_effective_cache_size || exit $? -set_maintenance_work_mem || exit $? -set_checkpoint_segments || exit $? -set_checkpoint_completion_target -set_wal_buffers -set_default_statistics_target || exit $? -set_random_page_cost || exit $? -set_effective_io_concurrency || exit $? -set_parallel_settings -set_work_mem || exit $? - -echo "# DB Version: "$db_version -echo "# OS Type: linux" -echo "# DB Type: "$db_type -echo "# Total Memory (RAM): "$(format_value $total_mem 1) -echo "# CPUs num: "$cpu_num -echo "# Connections num: "$conn_nb -echo "# Data Storage: "$storage_type -echo -echo "max_connections = "$conn_nb -echo "shared_buffers = "$(format_value $shared_buffers) -echo "effective_cache_size = "$(format_value $effective_cache_size) -echo "maintenance_work_mem = "$(format_value $maintenance_work_mem) -echo "checkpoint_completion_target = "$checkpoint_completion_target -echo "wal_buffers = "$(format_value $wal_buffers) -echo "default_statistics_target = "$default_statistics_target -echo "random_page_cost = "$random_page_cost -echo "effective_io_concurrency = "$effective_io_concurrency -echo "work_mem = "$(format_value $work_mem) -echo "min_wal_size = "$(format_value $min_wal_size) -echo "max_wal_size = "$(format_value $max_wal_size) -for key in "${!parallel_settings[@]}" -do - echo $key" = "${parallel_settings[$key]} -done -if [ ! -z ${checkpoint_segments+x}] -then - echo "checkpoint_segments = "$checkpoint_segments -fi - -unset set_parallel_settings + DB_TYPE="$t" + ;; + m) + m=$OPTARG + if [[ $m == *"MB"* ]]; then + ram=${m%"MB"} + if [ "$ram" -lt "512" ] || [ "$ram" -gt "9999" ]; then + _input_error "total memory in MB must be greater than or equal to 512MB and less than or equal to 9999MB" + fi + ram=$(( $ram*$KB )) + elif [[ $m == *"GB"* ]]; then + ram=${m%"GB"} + if [ "$ram" -lt "1" ] || [ "$ram" -gt "9999" ]; then + _input_error "total memory in GB must be greater than or equal to 1GB and less than or equal to 9999GB" + fi + ram=$(( $ram*$MB )) + else + _input_error "$m does not contain a valid unit identifier (use MB or GB)" + fi + TOTAL_MEM="$ram" + ;; + u) + u=$OPTARG + if [ "$u" -lt "1" ] || [ "$u" -gt "9999" ]; then + _input_error "CPU count must be greater than or equal to 1 and less than or equal to 9999" + fi + CPU_NUM=$u + ;; + c) + c=$OPTARG + if [ "$c" -lt "20" ] || [ "$c" -gt "9999" ]; then + _input_error "connections number must be greater than or equal to 1 and less than or equal to 9999" + fi + CONN_NUM=$c + ;; + s) + s=$OPTARG + if [ $s != "hdd" ] && \ + [ $s != "ssd" ] && \ + [ $s != "san" ]; then + _input_error "$s is not a valid storage type identifier" + fi + STORAGE_TYPE="$s" + ;; + *) + show_help >&2 + exit 2 + ;; + esac + done + + if [ "$DB_VERSION" -eq "0" ]; then + DB_VERSION=16 + fi + + if [ "$TOTAL_MEM" -eq "0" ]; then + TOTAL_MEM=$(get_total_ram) || exit $? + fi + + if [ "$CPU_NUM" -eq "0" ]; then + CPU_NUM=$(get_cpu_count) || exit $? + fi + + if [ -z "$DB_TYPE" ]; then + DB_TYPE="web" + fi + + if [ -z "$STORAGE_TYPE" ]; then + STORAGE_TYPE=$(get_disk_type) + fi + + if [ "$DB_VERSION" -eq "0" ]; then + DB_VERSION=16 + fi + + if [ $CONN_NUM -eq "0" ]; then + case $DB_TYPE in + "web") + CONN_NUM=200 + ;; + "oltp") + CONN_NUM=300 + ;; + "dw") + CONN_NUM=40 + ;; + "desktop") + CONN_NUM=20 + ;; + "mixed") + CONN_NUM=100 + ;; + *) + CONN_NUM=20 + ;; + esac + fi + + readonly DB_VERSION + readonly TOTAL_MEM + readonly CPU_NUM + readonly DB_TYPE + readonly STORAGE_TYPE + readonly DB_VERSION + readonly CONN_NUM + + set_db_default_values || exit $? + set_shared_buffers || exit $? + set_effective_cache_size || exit $? + set_maintenance_work_mem || exit $? + set_checkpoint_segments || exit $? + set_checkpoint_completion_target + set_wal_buffers + set_default_statistics_target || exit $? + set_random_page_cost || exit $? + set_effective_io_concurrency || exit $? + set_parallel_settings + set_work_mem || exit $? + + echo "# DB Version: "$DB_VERSION + echo "# OS Type: linux" + echo "# DB Type: "$DB_TYPE + echo "# Total Memory (RAM): "$(format_value $TOTAL_MEM 1) + echo "# CPUs num: "$CPU_NUM + echo "# Connections num: "$CONN_NUM + echo "# Data Storage: "$STORAGE_TYPE + echo + echo "max_connections = "$CONN_NUM + echo "shared_buffers = "$(format_value $shared_buffers) + echo "effective_cache_size = "$(format_value $effective_cache_size) + echo "maintenance_work_mem = "$(format_value $maintenance_work_mem) + echo "checkpoint_completion_target = "$checkpoint_completion_target + echo "wal_buffers = "$(format_value $wal_buffers) + echo "default_statistics_target = "$default_statistics_target + echo "random_page_cost = "$random_page_cost + echo "effective_io_concurrency = "$effective_io_concurrency + echo "work_mem = "$(format_value $work_mem) + echo "min_wal_size = "$(format_value $min_wal_size) + echo "max_wal_size = "$(format_value $max_wal_size) + + for key in "${!parallel_settings[@]}"; do + echo $key" = "${parallel_settings[$key]} + done + if [ ! -z ${checkpoint_segments+x} ]; then + echo "checkpoint_segments = "$checkpoint_segments + fi + + unset set_parallel_settings +} + +main "$@" diff --git a/test/expected_results/10_linux_web_8GB_8_1000_hdd.txt b/test/expected_results/10_linux_web_8GB_8_1000_hdd.txt index 92e38d6..47f5b36 100644 --- a/test/expected_results/10_linux_web_8GB_8_1000_hdd.txt +++ b/test/expected_results/10_linux_web_8GB_8_1000_hdd.txt @@ -19,5 +19,5 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 -max_parallel_workers = 8 max_parallel_workers_per_gather = 4 +max_parallel_workers = 8 diff --git a/test/expected_results/10_linux_web_8GB_8_1000_san.txt b/test/expected_results/10_linux_web_8GB_8_1000_san.txt index 51fd1bd..b05417e 100644 --- a/test/expected_results/10_linux_web_8GB_8_1000_san.txt +++ b/test/expected_results/10_linux_web_8GB_8_1000_san.txt @@ -19,5 +19,5 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 -max_parallel_workers = 8 max_parallel_workers_per_gather = 4 +max_parallel_workers = 8 diff --git a/test/expected_results/10_linux_web_8GB_8_1000_ssd.txt b/test/expected_results/10_linux_web_8GB_8_1000_ssd.txt index f8f97ec..f752792 100644 --- a/test/expected_results/10_linux_web_8GB_8_1000_ssd.txt +++ b/test/expected_results/10_linux_web_8GB_8_1000_ssd.txt @@ -19,5 +19,5 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 -max_parallel_workers = 8 max_parallel_workers_per_gather = 4 +max_parallel_workers = 8 diff --git a/test/expected_results/11_linux_web_8GB_8_1000_hdd.txt b/test/expected_results/11_linux_web_8GB_8_1000_hdd.txt index e60a407..a5d68a1 100644 --- a/test/expected_results/11_linux_web_8GB_8_1000_hdd.txt +++ b/test/expected_results/11_linux_web_8GB_8_1000_hdd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/11_linux_web_8GB_8_1000_san.txt b/test/expected_results/11_linux_web_8GB_8_1000_san.txt index 4d1e461..868e208 100644 --- a/test/expected_results/11_linux_web_8GB_8_1000_san.txt +++ b/test/expected_results/11_linux_web_8GB_8_1000_san.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/11_linux_web_8GB_8_1000_ssd.txt b/test/expected_results/11_linux_web_8GB_8_1000_ssd.txt index e6d8329..126d21f 100644 --- a/test/expected_results/11_linux_web_8GB_8_1000_ssd.txt +++ b/test/expected_results/11_linux_web_8GB_8_1000_ssd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/12_linux_web_8GB_8_1000_hdd.txt b/test/expected_results/12_linux_web_8GB_8_1000_hdd.txt index 62680a6..02b497b 100644 --- a/test/expected_results/12_linux_web_8GB_8_1000_hdd.txt +++ b/test/expected_results/12_linux_web_8GB_8_1000_hdd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/12_linux_web_8GB_8_1000_san.txt b/test/expected_results/12_linux_web_8GB_8_1000_san.txt index 5370a44..867dfeb 100644 --- a/test/expected_results/12_linux_web_8GB_8_1000_san.txt +++ b/test/expected_results/12_linux_web_8GB_8_1000_san.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/12_linux_web_8GB_8_1000_ssd.txt b/test/expected_results/12_linux_web_8GB_8_1000_ssd.txt index 994fb82..995f9ca 100644 --- a/test/expected_results/12_linux_web_8GB_8_1000_ssd.txt +++ b/test/expected_results/12_linux_web_8GB_8_1000_ssd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/13_linux_web_8GB_8_1000_hdd.txt b/test/expected_results/13_linux_web_8GB_8_1000_hdd.txt index e69ee5a..468d3d4 100644 --- a/test/expected_results/13_linux_web_8GB_8_1000_hdd.txt +++ b/test/expected_results/13_linux_web_8GB_8_1000_hdd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/13_linux_web_8GB_8_1000_san.txt b/test/expected_results/13_linux_web_8GB_8_1000_san.txt index f34648e..dfbf149 100644 --- a/test/expected_results/13_linux_web_8GB_8_1000_san.txt +++ b/test/expected_results/13_linux_web_8GB_8_1000_san.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/13_linux_web_8GB_8_1000_ssd.txt b/test/expected_results/13_linux_web_8GB_8_1000_ssd.txt index 08a9c48..68723bf 100644 --- a/test/expected_results/13_linux_web_8GB_8_1000_ssd.txt +++ b/test/expected_results/13_linux_web_8GB_8_1000_ssd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/14_linux_web_8GB_8_1000_hdd.txt b/test/expected_results/14_linux_web_8GB_8_1000_hdd.txt index cae331e..cb1fca5 100644 --- a/test/expected_results/14_linux_web_8GB_8_1000_hdd.txt +++ b/test/expected_results/14_linux_web_8GB_8_1000_hdd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/14_linux_web_8GB_8_1000_san.txt b/test/expected_results/14_linux_web_8GB_8_1000_san.txt index ddf738b..9b7b4f4 100644 --- a/test/expected_results/14_linux_web_8GB_8_1000_san.txt +++ b/test/expected_results/14_linux_web_8GB_8_1000_san.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/14_linux_web_8GB_8_1000_ssd.txt b/test/expected_results/14_linux_web_8GB_8_1000_ssd.txt index f88a45c..428e03d 100644 --- a/test/expected_results/14_linux_web_8GB_8_1000_ssd.txt +++ b/test/expected_results/14_linux_web_8GB_8_1000_ssd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/15_linux_web_8GB_8_1000_hdd.txt b/test/expected_results/15_linux_web_8GB_8_1000_hdd.txt index 6ae554a..10edf08 100644 --- a/test/expected_results/15_linux_web_8GB_8_1000_hdd.txt +++ b/test/expected_results/15_linux_web_8GB_8_1000_hdd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/15_linux_web_8GB_8_1000_san.txt b/test/expected_results/15_linux_web_8GB_8_1000_san.txt index 4e6b3de..04c3ce6 100644 --- a/test/expected_results/15_linux_web_8GB_8_1000_san.txt +++ b/test/expected_results/15_linux_web_8GB_8_1000_san.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/expected_results/15_linux_web_8GB_8_1000_ssd.txt b/test/expected_results/15_linux_web_8GB_8_1000_ssd.txt index e7f5203..00df6c7 100644 --- a/test/expected_results/15_linux_web_8GB_8_1000_ssd.txt +++ b/test/expected_results/15_linux_web_8GB_8_1000_ssd.txt @@ -19,6 +19,6 @@ work_mem = 524kB min_wal_size = 1GB max_wal_size = 4GB max_worker_processes = 8 +max_parallel_workers_per_gather = 4 max_parallel_workers = 8 max_parallel_maintenance_workers = 4 -max_parallel_workers_per_gather = 4 diff --git a/test/scripts/pull_testing_images.sh b/test/scripts/pull_testing_images.sh index 9722d49..42ea151 100644 --- a/test/scripts/pull_testing_images.sh +++ b/test/scripts/pull_testing_images.sh @@ -3,17 +3,21 @@ # This script pulls images from Docker Hub and run them # to do a simple check of PostgreSQL configuration +set -e + testing_image="esgn/pgtuned-testing" -tags=$(curl "https://registry.hub.docker.com/v2/repositories/$testing_image/tags?page_size=32" 2>/dev/null | jq -r '.results | .[] | .name') -for tag in $tags -do - image_name=$testing_image":"$tag - docker rmi $image_name &> /dev/null - docker pull -q $image_name - docker run -d -e POSTGRES_PASSWORD=secret --name pg-testing$tag $image_name - docker exec -ti pg-testing$tag bash -c "until pg_isready -q; do sleep 5; done" - docker exec -ti pg-testing$tag bash -c "cat /etc/apt/sources.list.d/pgdg.list" - docker exec -ti pg-testing$tag bash -c "cat /var/lib/postgresql/data/postgresql.conf" > pg-testing$tag".txt" - docker stop pg-testing$tag &> /dev/null && docker rm pg-testing$tag &> /dev/null +# get all available tags from docker hub registry +tags=$(curl "https://registry.hub.docker.com/v2/repositories/$testing_image/tags?page_size=100" 2>/dev/null | jq -r '.results | .[] | .name') + +# Iterate over tags +for tag in $tags; do + image_name=$testing_image":"$tag + docker rmi --force $image_name &>/dev/null + docker pull -q $image_name + docker run -d -e POSTGRES_PASSWORD=secret --name pg-testing$tag $image_name + docker exec -ti pg-testing$tag bash -c "until pg_isready -q; do sleep 5; done" + # docker exec -ti pg-testing$tag bash -c "cat /etc/apt/sources.list.d/pgdg.list" + docker exec -ti pg-testing$tag bash -c "cat /var/lib/postgresql/data/postgresql.conf" >pg-testing$tag".txt" + docker stop pg-testing$tag &>/dev/null && docker rm --force pg-testing$tag &>/dev/null done diff --git a/test/scripts/test_pgtune.sh b/test/scripts/test_pgtune.sh index b43f6ae..5bdf320 100644 --- a/test/scripts/test_pgtune.sh +++ b/test/scripts/test_pgtune.sh @@ -6,7 +6,7 @@ test_files_dir="../expected_results/" pgtuned_script="../../pgtune.sh" pg_versions="16 15 14 13 12 11 10 9.6 9.5" -db_types="web oltp dw desktop mixed" +# db_types="web oltp dw desktop mixed" total_mem=8GB cpu_count=8 max_conn=1000