From 9e03addd609b5ec5db4f9ab5d3a937a5e816ca43 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Fri, 3 Nov 2023 14:48:44 +0100 Subject: [PATCH] add toggle to enable grouping log by date --- postgres-appliance/scripts/configure_spilo.py | 3 +++ postgres-appliance/scripts/upload_pg_log_to_s3.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/postgres-appliance/scripts/configure_spilo.py b/postgres-appliance/scripts/configure_spilo.py index 00f460c1e..3195327fb 100755 --- a/postgres-appliance/scripts/configure_spilo.py +++ b/postgres-appliance/scripts/configure_spilo.py @@ -579,6 +579,7 @@ def get_placeholders(provider): placeholders.setdefault('CLONE_TARGET_TIME', '') placeholders.setdefault('CLONE_TARGET_INCLUSIVE', True) + placeholders.setdefault('LOG_GROUP_BY_DATE', False) placeholders.setdefault('LOG_SHIP_SCHEDULE', '1 0 * * *') placeholders.setdefault('LOG_S3_BUCKET', '') placeholders.setdefault('LOG_S3_ENDPOINT', '') @@ -758,6 +759,8 @@ def write_log_environment(placeholders): log_env['LOG_AWS_REGION'] = aws_region log_s3_key = 'spilo/{LOG_BUCKET_SCOPE_PREFIX}{SCOPE}{LOG_BUCKET_SCOPE_SUFFIX}/log/'.format(**log_env) + if os.getenv('LOG_GROUP_BY_DATE'): + log_s3_key += '{DATE}/' log_s3_key += placeholders['instance_data']['id'] log_env['LOG_S3_KEY'] = log_s3_key diff --git a/postgres-appliance/scripts/upload_pg_log_to_s3.py b/postgres-appliance/scripts/upload_pg_log_to_s3.py index 259ca598a..dd6e1fd83 100755 --- a/postgres-appliance/scripts/upload_pg_log_to_s3.py +++ b/postgres-appliance/scripts/upload_pg_log_to_s3.py @@ -48,6 +48,8 @@ def upload_to_s3(local_file_path): bucket = s3.Bucket(bucket_name) key_name = os.path.join(os.getenv('LOG_S3_KEY'), os.path.basename(local_file_path)) + if os.getenv('LOG_GROUP_BY_DATE'): + key_name = key_name.format(**{'DATE': os.path.basename(local_file_path).split('.')[0]}) chunk_size = 52428800 # 50 MiB config = TransferConfig(multipart_threshold=chunk_size, multipart_chunksize=chunk_size)