From 06fc26d1505e265dd60d0f3390c3f61131686cb5 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Mon, 29 Jan 2024 15:27:00 +0100 Subject: [PATCH 1/8] Switch to IMDSv2 --- .../build_scripts/patroni_wale.sh | 3 +-- postgres-appliance/scripts/callback_aws.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/postgres-appliance/build_scripts/patroni_wale.sh b/postgres-appliance/build_scripts/patroni_wale.sh index a476954ec..43b64393c 100644 --- a/postgres-appliance/build_scripts/patroni_wale.sh +++ b/postgres-appliance/build_scripts/patroni_wale.sh @@ -26,7 +26,6 @@ if [ "$DEMO" != "true" ]; then python3-etcd \ python3-consul \ python3-kazoo \ - python3-boto \ python3-boto3 \ python3-botocore \ python3-cachetools \ @@ -44,7 +43,7 @@ if [ "$DEMO" != "true" ]; then 'git+https://github.com/zalando/pg_view.git@master#egg=pg-view' # https://github.com/wal-e/wal-e/issues/318 - sed -i 's/^\( for i in range(0,\) num_retries):.*/\1 100):/g' /usr/lib/python3/dist-packages/boto/utils.py + sed -i 's/^\( for i in range(0,\) num_retries):.*/\1 100):/g' /usr/lib/python3/dist-packages/boto3/utils.py else EXTRAS="" fi diff --git a/postgres-appliance/scripts/callback_aws.py b/postgres-appliance/scripts/callback_aws.py index 971ac7684..382757be1 100755 --- a/postgres-appliance/scripts/callback_aws.py +++ b/postgres-appliance/scripts/callback_aws.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -import boto.ec2 -import boto.utils +import boto3 import logging import sys import time +import requests logger = logging.getLogger(__name__) @@ -15,7 +15,7 @@ def wrapped(*args, **kwargs): while True: try: return func(*args, **kwargs) - except boto.exception.BotoServerError as e: + except boto3.exceptions.Boto3Error as e: if count >= 10 or str(e.error_code) not in ('Throttling', 'RequestLimitExceeded'): raise logger.info('Throttling AWS API requests...') @@ -24,10 +24,13 @@ def wrapped(*args, **kwargs): return wrapped - def get_instance_metadata(): - return boto.utils.get_instance_identity()['document'] + response = requests.put('http://169.254.169.254/latest/api/token', headers={'X-aws-ec2-metadata-token-ttl-seconds': '60'}) + token = response.text + headers = {'X-aws-ec2-metadata-token': token} + instance_id = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', headers=headers) + return instance_id.json() @retry def associate_address(ec2, allocation_id, instance_id): @@ -63,7 +66,10 @@ def main(): instance_id = metadata['instanceId'] - ec2 = boto.ec2.connect_to_region(metadata['region']) + ec2 = boto3.client( + service_name='ec2', + region_name=metadata['region'] + ) if argc == 5 and role in ('master', 'standby_leader') and action in ('on_start', 'on_role_change'): associate_address(ec2, sys.argv[1], instance_id) From b6a13200b14354ede86261fd5bf14d868688f545 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Mon, 29 Jan 2024 15:35:39 +0100 Subject: [PATCH 2/8] fix formatting --- postgres-appliance/scripts/callback_aws.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/postgres-appliance/scripts/callback_aws.py b/postgres-appliance/scripts/callback_aws.py index 382757be1..b9eb8a6b1 100755 --- a/postgres-appliance/scripts/callback_aws.py +++ b/postgres-appliance/scripts/callback_aws.py @@ -24,14 +24,19 @@ def wrapped(*args, **kwargs): return wrapped + def get_instance_metadata(): - response = requests.put('http://169.254.169.254/latest/api/token', headers={'X-aws-ec2-metadata-token-ttl-seconds': '60'}) + response = requests.put( + url='http://169.254.169.254/latest/api/token', + headers={'X-aws-ec2-metadata-token-ttl-seconds': '60'} + ) token = response.text headers = {'X-aws-ec2-metadata-token': token} instance_id = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', headers=headers) return instance_id.json() + @retry def associate_address(ec2, allocation_id, instance_id): return ec2.associate_address(instance_id=instance_id, allocation_id=allocation_id, allow_reassociation=True) From 464b6f8b43200c4db2eb4c2cc683d5736cb0179b Mon Sep 17 00:00:00 2001 From: idanovinda Date: Mon, 29 Jan 2024 15:56:04 +0100 Subject: [PATCH 3/8] refactor --- postgres-appliance/scripts/callback_aws.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/postgres-appliance/scripts/callback_aws.py b/postgres-appliance/scripts/callback_aws.py index b9eb8a6b1..2d196f460 100755 --- a/postgres-appliance/scripts/callback_aws.py +++ b/postgres-appliance/scripts/callback_aws.py @@ -31,10 +31,11 @@ def get_instance_metadata(): headers={'X-aws-ec2-metadata-token-ttl-seconds': '60'} ) token = response.text - headers = {'X-aws-ec2-metadata-token': token} - - instance_id = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', headers=headers) - return instance_id.json() + instance_identity = requests.get( + url='http://169.254.169.254/latest/dynamic/instance-identity/document', + headers={'X-aws-ec2-metadata-token': token} + ) + return instance_identity.json() @retry From a1c20b58cbd0e19e5b98f584422a1dff8efa161b Mon Sep 17 00:00:00 2001 From: idanovinda Date: Wed, 7 Feb 2024 17:35:17 +0100 Subject: [PATCH 4/8] update boto3 functions and revert patroni_wale.sh --- .../build_scripts/patroni_wale.sh | 3 +- postgres-appliance/scripts/callback_aws.py | 28 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/postgres-appliance/build_scripts/patroni_wale.sh b/postgres-appliance/build_scripts/patroni_wale.sh index 43b64393c..571a751ae 100644 --- a/postgres-appliance/build_scripts/patroni_wale.sh +++ b/postgres-appliance/build_scripts/patroni_wale.sh @@ -27,6 +27,7 @@ if [ "$DEMO" != "true" ]; then python3-consul \ python3-kazoo \ python3-boto3 \ + python3-boto \ python3-botocore \ python3-cachetools \ python3-cffi \ @@ -43,7 +44,7 @@ if [ "$DEMO" != "true" ]; then 'git+https://github.com/zalando/pg_view.git@master#egg=pg-view' # https://github.com/wal-e/wal-e/issues/318 - sed -i 's/^\( for i in range(0,\) num_retries):.*/\1 100):/g' /usr/lib/python3/dist-packages/boto3/utils.py + sed -i 's/^\( for i in range(0,\) num_retries):.*/\1 100):/g' /usr/lib/python3/dist-packages/boto/utils.py else EXTRAS="" fi diff --git a/postgres-appliance/scripts/callback_aws.py b/postgres-appliance/scripts/callback_aws.py index 2d196f460..e48a5d79b 100755 --- a/postgres-appliance/scripts/callback_aws.py +++ b/postgres-appliance/scripts/callback_aws.py @@ -40,22 +40,22 @@ def get_instance_metadata(): @retry def associate_address(ec2, allocation_id, instance_id): - return ec2.associate_address(instance_id=instance_id, allocation_id=allocation_id, allow_reassociation=True) + return ec2.associate_address(InstanceId=instance_id, AllocationId=allocation_id, AllowReassociation=True) @retry def tag_resource(ec2, resource_id, tags): - return ec2.create_tags([resource_id], tags) + return ec2.create_tags(Resources=[resource_id], Tags=tags) @retry def list_volumes(ec2, instance_id): - return ec2.get_all_volumes(filters={'attachment.instance-id': instance_id}) + return ec2.describe_volumes(Filters=[{'Name': 'attachment.instance-id', 'Values': [instance_id]}]) @retry def get_instance(ec2, instance_id): - return ec2.get_only_instances([instance_id])[0] + return ec2.describe_instances(InstanceIds=[instance_id])['Reservations'][0]['Instances'][0] def main(): @@ -72,10 +72,7 @@ def main(): instance_id = metadata['instanceId'] - ec2 = boto3.client( - service_name='ec2', - region_name=metadata['region'] - ) + ec2 = boto3.client('ec2', region_name=metadata['region']) if argc == 5 and role in ('master', 'standby_leader') and action in ('on_start', 'on_role_change'): associate_address(ec2, sys.argv[1], instance_id) @@ -88,15 +85,16 @@ def main(): tags.update({'Instance': instance_id}) volumes = list_volumes(ec2, instance_id) - for v in volumes: - if 'Name' in v.tags: + for v in volumes['Volumes']: + if any(tag['Key'] == 'Name' for tag in v.get('Tags', [])): tags_to_update = tags else: - if v.attach_data.device == instance.root_device_name: - volume_device = 'root' - else: - volume_device = 'data' - tags_to_update = dict(tags, Name='spilo_{}_{}'.format(cluster, volume_device)) + for attachment in v['Attachments']: + if attachment['Device'] == instance.get('RootDeviceName'): + volume_device = 'root' + else: + volume_device = 'data' + tags_to_update = dict(tags, Name='spilo_{}_{}'.format(cluster, volume_device)) tag_resource(ec2, v.id, tags_to_update) From 760f75b7c16f9e47cd9e60fd6c625063d9d70522 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Wed, 7 Feb 2024 17:38:09 +0100 Subject: [PATCH 5/8] use imdsv2 in configure_spilo.py --- postgres-appliance/scripts/configure_spilo.py | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/postgres-appliance/scripts/configure_spilo.py b/postgres-appliance/scripts/configure_spilo.py index 3195327fb..597d11a24 100755 --- a/postgres-appliance/scripts/configure_spilo.py +++ b/postgres-appliance/scripts/configure_spilo.py @@ -399,12 +399,23 @@ def get_provider(): try: logging.info("Figuring out my environment (Google? AWS? Openstack? Local?)") - r = requests.get('http://169.254.169.254', timeout=2) + response = requests.put( + url='http://169.254.169.254/latest/api/token', + headers={'X-aws-ec2-metadata-token-ttl-seconds': '60'} + ) + token = response.text + r = requests.get( + url='http://169.254.169.254', + headers={'X-aws-ec2-metadata-token': token} + ) if r.headers.get('Metadata-Flavor', '') == 'Google': return PROVIDER_GOOGLE else: # accessible on Openstack, will fail on AWS - r = requests.get('http://169.254.169.254/openstack/latest/meta_data.json') + r = requests.get( + url='http://169.254.169.254/openstack/latest/meta_data.json', + headers={'X-aws-ec2-metadata-token': token} + ) if r.ok: # make sure the response is parsable - https://github.com/Azure/aad-pod-identity/issues/943 and # https://github.com/zalando/spilo/issues/542 @@ -412,7 +423,10 @@ def get_provider(): return PROVIDER_OPENSTACK # is accessible from both AWS and Openstack, Possiblity of misidentification if previous try fails - r = requests.get('http://169.254.169.254/latest/meta-data/ami-id') + r = requests.get( + url='http://169.254.169.254/latest/meta-data/ami-id', + headers={'X-aws-ec2-metadata-token': token} + ) return PROVIDER_AWS if r.ok else PROVIDER_UNSUPPORTED except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): logging.info("Could not connect to 169.254.169.254, assuming local Docker setup") @@ -451,7 +465,15 @@ def get_instance_metadata(provider): # Try get IP via OpenStack EC2-compatible API, if can't then fail back to auto-discovered one. metadata['id'] = openstack_metadata['uuid'] url = 'http://169.254.169.254/2009-04-04/meta-data' - r = requests.get(url) + response = requests.put( + url='http://169.254.169.254/latest/api/token', + headers={'X-aws-ec2-metadata-token-ttl-seconds': '60'} + ) + token = response.text + r = requests.get( + url=url, + headers={'X-aws-ec2-metadata-token': token} + ) if r.ok: mapping.update({'ip': 'local-ipv4', 'id': 'instance-id'}) else: From 42787bf36da35dde609c0e9bbe47bb4360be052a Mon Sep 17 00:00:00 2001 From: idanovinda Date: Thu, 8 Feb 2024 11:50:35 +0100 Subject: [PATCH 6/8] remove manual retry function --- .../build_scripts/patroni_wale.sh | 2 +- postgres-appliance/scripts/callback_aws.py | 31 ++++++------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/postgres-appliance/build_scripts/patroni_wale.sh b/postgres-appliance/build_scripts/patroni_wale.sh index 571a751ae..a476954ec 100644 --- a/postgres-appliance/build_scripts/patroni_wale.sh +++ b/postgres-appliance/build_scripts/patroni_wale.sh @@ -26,8 +26,8 @@ if [ "$DEMO" != "true" ]; then python3-etcd \ python3-consul \ python3-kazoo \ - python3-boto3 \ python3-boto \ + python3-boto3 \ python3-botocore \ python3-cachetools \ python3-cffi \ diff --git a/postgres-appliance/scripts/callback_aws.py b/postgres-appliance/scripts/callback_aws.py index e48a5d79b..b9b7c8f2a 100755 --- a/postgres-appliance/scripts/callback_aws.py +++ b/postgres-appliance/scripts/callback_aws.py @@ -1,30 +1,14 @@ #!/usr/bin/env python +from botocore.config import Config import boto3 import logging import sys -import time import requests logger = logging.getLogger(__name__) -def retry(func): - def wrapped(*args, **kwargs): - count = 0 - while True: - try: - return func(*args, **kwargs) - except boto3.exceptions.Boto3Error as e: - if count >= 10 or str(e.error_code) not in ('Throttling', 'RequestLimitExceeded'): - raise - logger.info('Throttling AWS API requests...') - time.sleep(2 ** count * 0.5) - count += 1 - - return wrapped - - def get_instance_metadata(): response = requests.put( url='http://169.254.169.254/latest/api/token', @@ -38,22 +22,18 @@ def get_instance_metadata(): return instance_identity.json() -@retry def associate_address(ec2, allocation_id, instance_id): return ec2.associate_address(InstanceId=instance_id, AllocationId=allocation_id, AllowReassociation=True) -@retry def tag_resource(ec2, resource_id, tags): return ec2.create_tags(Resources=[resource_id], Tags=tags) -@retry def list_volumes(ec2, instance_id): return ec2.describe_volumes(Filters=[{'Name': 'attachment.instance-id', 'Values': [instance_id]}]) -@retry def get_instance(ec2, instance_id): return ec2.describe_instances(InstanceIds=[instance_id])['Reservations'][0]['Instances'][0] @@ -72,7 +52,14 @@ def main(): instance_id = metadata['instanceId'] - ec2 = boto3.client('ec2', region_name=metadata['region']) + config = Config( + region_name=metadata['region'], + retries={ + 'max_attempts': 10, + 'mode': 'standard' + } + ) + ec2 = boto3.client('ec2', config=config) if argc == 5 and role in ('master', 'standby_leader') and action in ('on_start', 'on_role_change'): associate_address(ec2, sys.argv[1], instance_id) From 33fc3015de9ee7f81c7eaf263e032c235e6498d3 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Tue, 13 Feb 2024 09:46:05 +0100 Subject: [PATCH 7/8] update tag formatting --- postgres-appliance/scripts/callback_aws.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postgres-appliance/scripts/callback_aws.py b/postgres-appliance/scripts/callback_aws.py index b9b7c8f2a..5082ae52d 100755 --- a/postgres-appliance/scripts/callback_aws.py +++ b/postgres-appliance/scripts/callback_aws.py @@ -66,10 +66,10 @@ def main(): instance = get_instance(ec2, instance_id) - tags = {'Role': role} + tags = [{'Key': 'Role', 'Value': role}] tag_resource(ec2, instance_id, tags) - tags.update({'Instance': instance_id}) + tags.append({'Key': 'Instance', 'Value': instance_id}) volumes = list_volumes(ec2, instance_id) for v in volumes['Volumes']: @@ -81,9 +81,9 @@ def main(): volume_device = 'root' else: volume_device = 'data' - tags_to_update = dict(tags, Name='spilo_{}_{}'.format(cluster, volume_device)) + tags_to_update = tags + [{'Key': 'Name', 'Value': 'spilo_{}_{}'.format(cluster, volume_device)}] - tag_resource(ec2, v.id, tags_to_update) + tag_resource(ec2, v.get('VolumeId'), tags_to_update) if __name__ == '__main__': From e3f3146e0b4b5d2235fa4a42f9d527e164d21c92 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Mon, 8 Apr 2024 15:42:30 +0200 Subject: [PATCH 8/8] apply feedback --- postgres-appliance/scripts/configure_spilo.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/postgres-appliance/scripts/configure_spilo.py b/postgres-appliance/scripts/configure_spilo.py index 597d11a24..b4301c159 100755 --- a/postgres-appliance/scripts/configure_spilo.py +++ b/postgres-appliance/scripts/configure_spilo.py @@ -412,10 +412,7 @@ def get_provider(): return PROVIDER_GOOGLE else: # accessible on Openstack, will fail on AWS - r = requests.get( - url='http://169.254.169.254/openstack/latest/meta_data.json', - headers={'X-aws-ec2-metadata-token': token} - ) + r = requests.get('http://169.254.169.254/openstack/latest/meta_data.json') if r.ok: # make sure the response is parsable - https://github.com/Azure/aad-pod-identity/issues/943 and # https://github.com/zalando/spilo/issues/542 @@ -465,15 +462,7 @@ def get_instance_metadata(provider): # Try get IP via OpenStack EC2-compatible API, if can't then fail back to auto-discovered one. metadata['id'] = openstack_metadata['uuid'] url = 'http://169.254.169.254/2009-04-04/meta-data' - response = requests.put( - url='http://169.254.169.254/latest/api/token', - headers={'X-aws-ec2-metadata-token-ttl-seconds': '60'} - ) - token = response.text - r = requests.get( - url=url, - headers={'X-aws-ec2-metadata-token': token} - ) + r = requests.get(url) if r.ok: mapping.update({'ip': 'local-ipv4', 'id': 'instance-id'}) else: