From bd9008671ac3ed4572c91049db75a77ff114dcca Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Wed, 29 Mar 2017 10:58:32 +0100 Subject: [PATCH 1/5] Correct 9.6 to 9.1 in examples --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d58126c..2345c9e 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,8 @@ None ### Postgres 9.1 (with changes to kernel SHM settings and setting a max_connections) ``` vars: - postgresql_version: 9.6 - postgresql_conf_directory: /etc/postgresql/9.6 + postgresql_version: 9.1 + postgresql_conf_directory: /etc/postgresql/9.1 postgresql_tune_db_type: web postgresql_tune_total_memory: "{{ ansible_memtotal_mb }}MB" postgresql_tune_sysctl_file: /etc/sysctl.d/99-postgresql-tune.conf From e23443d0a895d11fa2dc5da5be0748debcbdb224 Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Wed, 29 Mar 2017 11:08:09 +0100 Subject: [PATCH 2/5] Add the option to specific a percentage of memory to use --- README.md | 23 +++++++++++++++++++++++ postgresql_tune.py | 22 +++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2345c9e..f71094d 100644 --- a/README.md +++ b/README.md @@ -61,3 +61,26 @@ None sysctl_file: "{{ postgresql_postgresql_tune_sysctl_file }}" ``` + +### Postgresql 9.6 ( using only 25% of the memory ) +``` + vars: + postgresql_version: 9.6 + postgresql_conf_directory: /etc/postgresql/9.6 + postgresql_tune_db_type: web + postgresql_tune_total_memory: "{{ ansible_memtotal_mb }}MB" + postgresql_tune_total_memory_percentage: 25 + + tasks: + - name: Tune Postgresql + postgresql_tune: + db_version: "{{ postgresql_version }}" + db_type: "{{ postgresql_tune_db_type }}" + total_memory: "{{ postgresql_tune_total_memory }}" + total_memory_percentage: "{{ postgresql_tune_total_memory_percentage }}" + postgresql_file: "{{ postgresql_conf_directory }}/conf.d/99-postgresql-tune.conf" + + +``` + + diff --git a/postgresql_tune.py b/postgresql_tune.py index 7503e01..baf6ac7 100644 --- a/postgresql_tune.py +++ b/postgresql_tune.py @@ -66,6 +66,13 @@ required: True default: null + total_memory_percentage: + description + - Tune based on a percentage of the total_memory. + Format: 1-100 + required: False + default: 100 + max_connections: description - Maximum number of PostgreSQL client's connections. @@ -301,6 +308,12 @@ def tune(data): const_for_size = CONST_SIZE[memory_arg[-2:]] total_memory = mem_in_size * const_for_size + total_memory_percentage = int(data["total_memory_percentage"]) + total_memory_original = total_memory + + if total_memory_percentage != 100: + total_memory = ( total_memory * total_memory_percentage / 100 ) + ### POSTGRESQL CONFIGURATION config["postgresql"] = postgres_settings( db_version, @@ -315,7 +328,9 @@ def tune(data): # document some key parameters in the on server config file confile.write("# pgtune db_version = " + str(db_version) + "\n") confile.write("# pgtune db_type = " + str(db_type) + "\n") - confile.write("# pgtune total_memory = " + str(total_memory / CONST_SIZE['GB']) + 'GB' + "\n") + confile.write("# pgtune total_memory = " + str(total_memory_original / CONST_SIZE['GB']) + 'GB' + "\n") + confile.write("# pgtune total_memory_percentage = " + data['total_memory_percentage'] + '%'+ "\n") + confile.write("# pgtune total_memory allocated = " + str(total_memory / CONST_SIZE['GB']) + 'GB' + "\n") for k,v in config["postgresql"].items(): confile.writelines("{} = {}\n".format(k,v)) @@ -351,6 +366,11 @@ def main(): "required": True, "type": "str" }, + "total_memory_percentage": { + "required": False, + "type": "str", + "default": 100 + }, "max_connections": { "required": True, "type": "str" From fbfa21f4c9ebe7900de85a698cdbf226a06c762c Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Wed, 29 Mar 2017 12:34:52 +0100 Subject: [PATCH 3/5] Calculate system memory to make total_memory an optional parameter --- postgresql_tune.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/postgresql_tune.py b/postgresql_tune.py index baf6ac7..6154f69 100644 --- a/postgresql_tune.py +++ b/postgresql_tune.py @@ -63,8 +63,8 @@ description - Total memory usable for PostgreSQL on server. Format: (1-9999)('MB' | GB') - required: True - default: null + required: False + default: Calculated total_memory_percentage: description @@ -353,6 +353,12 @@ def tune(data): def main(): """Main entry point function""" + + import os + # tested on Linux and Mac el Capitan + calculated_mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') + default_memory = str(int(calculated_mem_bytes/(1024.**2))) + 'MB' + fields = { "db_version": { "required": True, @@ -363,8 +369,9 @@ def main(): "type": "str" }, "total_memory": { - "required": True, - "type": "str" + "required": False, + "type": "str", + "default": default_memory }, "total_memory_percentage": { "required": False, From da557905babb0f6b9ee248dce5e724215bdb7b32 Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Wed, 29 Mar 2017 13:14:30 +0100 Subject: [PATCH 4/5] Adding option to disable max_connections templating so higher limits can be used but not tuned to --- README.md | 27 +++++++++++++++++++++++++++ postgresql_tune.py | 18 ++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/README.md b/README.md index f71094d..9410a12 100644 --- a/README.md +++ b/README.md @@ -82,5 +82,32 @@ None ``` +### Postgresql 9.6 with total system memory automatically calculated and disabling any templating of max_connections +``` + vars: + postgresql_version: 9.6 + postgresql_conf_directory: /etc/postgresql/9.6 + postgresql_tune_db_type: web + + tasks: + - name: Tune Postgresql + postgresql_tune: + db_version: "{{ postgresql_version }}" + db_type: "{{ postgresql_tune_db_type }}" + postgresql_file: "{{ postgresql_conf_directory }}/conf.d/99-postgresql-tune.conf" + disable_max_connections: true + +``` + +This will tune Postgresql on the standard 'web' settings for max_connections but allow you to set a much higher max_connections in your main postgresql.conf + +A standard mechanism for detecting system memory is used rather than Ansible:- + +``` +os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') +``` + +(tested on Linux and Mac el Capitan) + diff --git a/postgresql_tune.py b/postgresql_tune.py index 6154f69..674d260 100644 --- a/postgresql_tune.py +++ b/postgresql_tune.py @@ -79,6 +79,15 @@ Format: integer number (ex.: 100) required: True default: null + + disable_max_connections: + description + - Stop max_connections being passed down to the generated Postgresql config. + Useful to tune to a certain number of connections but allow a higher max. + Format: false|true + required: false + default: false + ''' @@ -323,6 +332,11 @@ def tune(data): max_connections ) + disable_max_connections = data["disable_max_connections"] + + if disable_max_connections: + config["postgresql"].pop("max_connections", None) + # write configuration file with open(data["postgresql_file"], 'w') as confile: # document some key parameters in the on server config file @@ -386,6 +400,10 @@ def main(): "required": True, "type": "str" }, + "disable_max_connections": { + "required": False, + "type": "bool" + }, "sysctl_file": { "required": False, "type": "str" From b6ebc1dbdecd6dbd14a090cd2ebf54cbb88c5dd3 Mon Sep 17 00:00:00 2001 From: TheDumbTechGuy Date: Wed, 19 Apr 2017 18:21:36 +0000 Subject: [PATCH 5/5] Added change detection and fixed invalid division - detects changes in config files and reports accordingly - adds check for non set sysctl_file value - creates folders if they don't exist - fixes an issue where total memory reported in config header is 0 due to available ram below 1GB --- postgresql_tune.py | 56 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/postgresql_tune.py b/postgresql_tune.py index 674d260..f938f6e 100644 --- a/postgresql_tune.py +++ b/postgresql_tune.py @@ -303,6 +303,32 @@ def format_config(config): return "\n".join(["{0} = {1}".format(k, v) for k, v in config.iteritems()]) +def create_dirs(file_path): + if not os.path.exists(os.path.dirname(file_path)): + try: + os.makedirs(os.path.dirname(file_path)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + +def file_exists(file_path): + if not file_path: + return False + elif not os.path.isfile(file_path): + return False + else: + return True + + +def md5(file_path): + hash_md5 = hashlib.md5() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + def tune(data): """Write PostgreSQL and Linux kernel configuration files""" config = {} @@ -316,9 +342,9 @@ def tune(data): mem_in_size = int(memory_arg[:-2]) const_for_size = CONST_SIZE[memory_arg[-2:]] total_memory = mem_in_size * const_for_size + total_memory_original = total_memory total_memory_percentage = int(data["total_memory_percentage"]) - total_memory_original = total_memory if total_memory_percentage != 100: total_memory = ( total_memory * total_memory_percentage / 100 ) @@ -336,18 +362,24 @@ def tune(data): if disable_max_connections: config["postgresql"].pop("max_connections", None) + # write configuration file + old_postgresql_file_hash = md5(data["postgresql_file"]) if file_exists(data["postgresql_file"]) else None + + create_dirs(data["postgresql_file"]) with open(data["postgresql_file"], 'w') as confile: # document some key parameters in the on server config file confile.write("# pgtune db_version = " + str(db_version) + "\n") confile.write("# pgtune db_type = " + str(db_type) + "\n") - confile.write("# pgtune total_memory = " + str(total_memory_original / CONST_SIZE['GB']) + 'GB' + "\n") + confile.write("# pgtune total_memory = " + str(total_memory_original / float(CONST_SIZE['GB'])) + 'GB' + "\n") confile.write("# pgtune total_memory_percentage = " + data['total_memory_percentage'] + '%'+ "\n") - confile.write("# pgtune total_memory allocated = " + str(total_memory / CONST_SIZE['GB']) + 'GB' + "\n") + confile.write("# pgtune total_memory allocated = " + str(total_memory / float(CONST_SIZE['GB'])) + 'GB' + "\n") for k,v in config["postgresql"].items(): confile.writelines("{} = {}\n".format(k,v)) + new_postgresql_file_hash = md5(data["postgresql_file"]) + ### KERNEL CONFIGURATION config["kernel"] = kernel_settings( @@ -358,11 +390,19 @@ def tune(data): ) # write configuration file - with open(data["sysctl_file"], 'w') as confile: - for k,v in config["kernel"].items(): - confile.writelines("{} = {}\n".format(k,v)) + old_sysctl_file_hash = None + new_sysctl_file_hash = None + if bool(data["sysctl_file"]): + old_sysctl_file_hash = md5(data["sysctl_file"]) if file_exists(data["sysctl_file"]) else None + + create_dirs(data["sysctl_file"]) + with open(data["sysctl_file"], 'w') as confile: + for k,v in config["kernel"].items(): + confile.writelines("{} = {}\n".format(k,v)) + + new_sysctl_file_hash = md5(data["postgresql_file"]) - return True, config + return (old_postgresql_file_hash != new_postgresql_file_hash, config) or (old_sysctl_file_hash != new_sysctl_file_hash) def main(): @@ -412,7 +452,7 @@ def main(): module = AnsibleModule(argument_spec=fields) has_changed, config = tune(module.params) - module.exit_json(changed=True, config=config) + module.exit_json(changed=has_changed, config=config) if __name__ == '__main__':