From e84f8f46355347fac38a290cb7a15cb015ed4915 Mon Sep 17 00:00:00 2001 From: Tarun Menon <64295670+tarunmenon95@users.noreply.github.com> Date: Wed, 26 Apr 2023 10:26:14 +1000 Subject: [PATCH] Merge pull request #146 from base2Services/develop (#147) * exlude docdb from rds cluster backups * Feature/unit tests refactor (#146) * Initial add * Disabled non docdb tests * Disabled non doc db stack resources * Fixed up set up test func * Changed waiter to rds * Fixed client call * Enable docdb pull test * Fixed bugs * Fixed typo * Fixed share id * Fixed docdb pull test * Disable docdb, enable rds instance * Fixed rds waiter * Enabled cloudformation for RDS Instance * Updated refactored unit tests * Removed test teardowns * Moved cleanup before assertion * Added fix for rds cluster docdb cleanup * Refactored unit tests * Re-enabled stack create * Removed bucket delete * Moved s3Upload outside script block in pipeline * Moved s3upload back into script block --- Jenkinsfile | 5 +- README.md | 3 + shelvery/rds_cluster_backup.py | 5 + shelvery_tests/cleanup_functions.py | 134 ++---- shelvery_tests/conftest.py | 8 +- shelvery_tests/docdb_integration_test.py | 220 ++++----- shelvery_tests/docdb_pull_test.py | 69 ++- shelvery_tests/ebs_integration_test.py | 262 ++++++----- shelvery_tests/ebs_pull_test.py | 61 ++- shelvery_tests/ec2ami_integration_test.py | 290 ++++++------ shelvery_tests/ec2ami_pull_test.py | 61 +-- .../rds_cluster_integration_test.py | 217 +++++---- shelvery_tests/rds_cluster_pull_test.py | 67 ++- shelvery_tests/rds_integration_test.py | 233 +++++---- shelvery_tests/rds_pull_test.py | 72 ++- shelvery_tests/resources.py | 21 + shelvery_tests/test_functions.py | 441 +----------------- shelvery_tests/zname_transformation_test.py | 6 - 18 files changed, 913 insertions(+), 1262 deletions(-) create mode 100644 shelvery_tests/resources.py diff --git a/Jenkinsfile b/Jenkinsfile index 9b5ba5c..010eb25 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -97,9 +97,10 @@ pipeline { def safebranch = env.BRANCH_NAME.replace("/", "_") def releaseFileName = env.BRANCH_NAME == 'master' ? fileName : fileName.replace('.tar.gz',"-${safebranch}.tar.gz") env["SHELVERY_S3_RELEASE"] = "https://${env.SHELVERY_DIST_BUCKET}.s3.amazonaws.com/release/${releaseFileName}" - s3Upload(bucket: env.SHELVERY_DIST_BUCKET, file: "dist/${fileName}", path: "release/${releaseFileName}") - } + + } + } post { success { diff --git a/README.md b/README.md index 3aafb67..d2aa9f1 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,9 @@ backups Note that when copying to a new key, the shelvery requires access to both the new key and the original key. The full KMS key ARN should be supplied. +- `shelvery_reencrypt_kms_key_id` - when it is required to re-encrypt a RDS instance or cluster snapshot with a different KMS key + than that of the existing resource. Specify the ARN of the KMS Key. + ### Configuration Priority 0: Sensible defaults ```text diff --git a/shelvery/rds_cluster_backup.py b/shelvery/rds_cluster_backup.py index 87228f6..f88cca7 100644 --- a/shelvery/rds_cluster_backup.py +++ b/shelvery/rds_cluster_backup.py @@ -1,3 +1,4 @@ +from tracemalloc import Snapshot import boto3 from shelvery.runtime_config import RuntimeConfig @@ -254,12 +255,14 @@ def get_all_clusters(self, rds_client): db_clusters = [] # temporary list of api models, as calls are batched temp_clusters = rds_client.describe_db_clusters() + db_clusters.extend(temp_clusters['DBClusters']) # collect database instances while 'Marker' in temp_clusters: temp_clusters = rds_client.describe_db_clusters(Marker=temp_clusters['Marker']) db_clusters.extend(temp_clusters['DBClusters']) + db_clusters = [cluster for cluster in db_clusters if cluster.get('Engine') != 'docdb'] return db_clusters def get_shelvery_backups_only(self, all_snapshots, backup_tag_prefix, rds_client): @@ -301,6 +304,8 @@ def collect_all_snapshots(self, rds_client): self.logger.info(f"Collected {len(tmp_snapshots['DBClusterSnapshots'])} manual snapshots. Continuing collection...") tmp_snapshots = rds_client.describe_db_cluster_snapshots(SnapshotType='manual', Marker=tmp_snapshots['Marker']) all_snapshots.extend(tmp_snapshots['DBClusterSnapshots']) + + all_snapshots = [snapshot for snapshot in all_snapshots if snapshot.get('Engine') != 'docdb'] self.logger.info(f"Collected {len(all_snapshots)} manual snapshots.") self.populate_snap_entity_resource(all_snapshots) diff --git a/shelvery_tests/cleanup_functions.py b/shelvery_tests/cleanup_functions.py index f4fdc50..757d75a 100644 --- a/shelvery_tests/cleanup_functions.py +++ b/shelvery_tests/cleanup_functions.py @@ -1,117 +1,53 @@ +from shelvery.documentdb_backup import ShelveryDocumentDbBackup +from shelvery.ebs_backup import ShelveryEBSBackup +from shelvery.ec2ami_backup import ShelveryEC2AMIBackup +from shelvery.rds_cluster_backup import ShelveryRDSClusterBackup +from shelvery.rds_backup import ShelveryRDSBackup from shelvery.aws_helper import AwsHelper +import boto3 +import os def cleanDocDBSnapshots(): print("Cleaning up DocDB Snapshots") - - docdbclient = AwsHelper.boto3_client('docdb', region_name='ap-southeast-2') - - snapshots = docdbclient.describe_db_cluster_snapshots( - DBClusterIdentifier='shelvery-test-docdb', - SnapshotType='Manual' - )['DBClusterSnapshots'] - - for snapshot in snapshots: - snapid = snapshot['DBClusterSnapshotIdentifier'] - - try: - print(f"Deleting snapshot: {snapid}") - docdbclient.delete_db_cluster_snapshot(DBClusterSnapshotIdentifier=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - + backups_engine = ShelveryDocumentDbBackup() + backups_engine.clean_backups() + def cleanRdsClusterSnapshots(): print("Cleaning up RDS Cluster Snapshots") - rdsclusterclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') - - snapshots = rdsclusterclient.describe_db_cluster_snapshots( - DBClusterIdentifier='shelvery-test-rds-cluster', - SnapshotType='Manual' - )['DBClusterSnapshots'] - - for snapshot in snapshots: - snapid = snapshot['DBClusterSnapshotIdentifier'] - - try: - print(f"Deleting snapshot: {snapid}") - rdsclusterclient.delete_db_cluster_snapshot(DBClusterSnapshotIdentifier=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") + backups_engine = ShelveryRDSClusterBackup() + backups_engine.clean_backups() def cleanRdsSnapshots(): print("Cleaning up RDS Snapshots") - rdsclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') - - snapshots = rdsclient.describe_db_snapshots( - DBInstanceIdentifier='shelvery-test-rds', - SnapshotType='Manual', - )['DBSnapshots'] - - for snapshot in snapshots: - snapid = snapshot['DBSnapshotIdentifier'] - - try: - print(f"Deleting snapshot: {snapid}") - rdsclient.delete_db_snapshot(DBSnapshotIdentifier=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") + backups_engine = ShelveryRDSBackup() + backups_engine.clean_backups() def cleanEC2Snapshots(): - #EC2 AMI - ec2client = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2') - sts = AwsHelper.boto3_client('sts') - id = str(sts.get_caller_identity()['Account']) - - snapshots = ec2client.describe_snapshots( - OwnerIds=[id] - )['Snapshots'] - - for snapshot in snapshots: - snapid = snapshot['SnapshotId'] - if 'Tags' in snapshot: - tags = snapshot['Tags'] - try: - name = [tag['Value'] for tag in tags if tag['Key'] == 'Name'][0] - if 'shelvery-test-ec2' in name: - print("Cleaning up EC2 AMI Snapshots") - ami_id = [tag['Value'] for tag in tags if tag['Key'] == 'shelvery:ami_id'][0] - if ami_id != []: - print(f"De-registering image: {ami_id}") - ec2client.deregister_image(ImageId=ami_id) - ec2client.delete_snapshot(SnapshotId=snapid) - print(f'Deleting EC2 snapshot: {snapid}') - if 'shelvery-test-ebs' in name: - print("Cleaning up EBS Snapshots") - print(f'Deleting EBS snapshot: {snapid}') - ec2client.delete_snapshot(SnapshotId=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - - else: - print(f'Deleting Untagged EC2 Snapshots') - if snapshot['VolumeId'] == 'vol-ffffffff' and 'Copied for' in snapshot['Description']: - - search_filter = [{'Name':'block-device-mapping.snapshot-id', - 'Values': [snapid], - 'Name':'tag:ResourceName', - 'Values':['shelvery-test-ec2'] - }] - - - - ami_id = ec2client.describe_images( - Filters=search_filter - )['Images'][0]['ImageId'] - try: - print(f"De-registering image: {ami_id}") - print(f'Deleting EC2 snapshot: {snapid}') - ec2client.deregister_image(ImageId=ami_id) - ec2client.delete_snapshot(SnapshotId=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") + print("Cleaning up EC2 AMI Snapshots") + backups_engine = ShelveryEC2AMIBackup() + backups_engine.clean_backups() + +def cleanEBSSnapshots(): + print("Cleaning up EBS Snapshots") + backups_engine = ShelveryEBSBackup() + backups_engine.clean_backups() + +def cleanS3Bucket(): + print("Cleaning S3 Bucket") + bucket_name = f"shelvery.data.{AwsHelper.local_account_id()}-ap-southeast-2.base2tools" + s3 = boto3.resource('s3') + bucket = s3.Bucket(bucket_name) + + # Delete all objects in the bucket + for obj in bucket.objects.all(): + obj.delete() def cleanupSnapshots(): + os.environ['shelvery_custom_retention_types'] = 'shortLived:1' + os.environ['shelvery_current_retention_type'] = 'shortLived' cleanDocDBSnapshots() cleanEC2Snapshots() + cleanEBSSnapshots() cleanRdsClusterSnapshots() cleanRdsSnapshots() \ No newline at end of file diff --git a/shelvery_tests/conftest.py b/shelvery_tests/conftest.py index 7e18d3f..dad1e1a 100644 --- a/shelvery_tests/conftest.py +++ b/shelvery_tests/conftest.py @@ -1,11 +1,10 @@ import pytest import boto3 import os -import time from shelvery.aws_helper import AwsHelper -from botocore.exceptions import ClientError, ValidationError, WaiterError +from botocore.exceptions import ClientError -from shelvery_tests.cleanup_functions import cleanupSnapshots +from shelvery_tests.cleanup_functions import cleanupSnapshots, cleanS3Bucket source_account = None destination_account = None @@ -59,6 +58,9 @@ def setup(request): #Cleanup any existing snapshots after stack is created cleanupSnapshots() + + #Cleanup S3 Bucket + cleanS3Bucket() def teardown(): print ("Initiating Teardown") diff --git a/shelvery_tests/docdb_integration_test.py b/shelvery_tests/docdb_integration_test.py index 3e301d5..6ba21e9 100644 --- a/shelvery_tests/docdb_integration_test.py +++ b/shelvery_tests/docdb_integration_test.py @@ -1,17 +1,14 @@ import sys -import traceback import unittest import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime from botocore.exceptions import WaiterError - -from shelvery_tests.test_functions import addBackupTags, clusterCleanupBackups, clusterShareBackups, compareBackups, initCleanup, initCreateBackups, initSetup, initShareBackups +from shelvery.engine import ShelveryEngine +from shelvery.runtime_config import RuntimeConfig +from shelvery_tests.resources import DOCDB_RESOURCE_NAME, ResourceClass +from shelvery_tests.test_functions import setup_source, compare_backups +from shelvery.documentdb_backup import ShelveryDocumentDbBackup +from shelvery.aws_helper import AwsHelper pwd = os.path.dirname(os.path.abspath(__file__)) @@ -21,41 +18,36 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.documentdb_backup import ShelveryDocumentDbBackup -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper -from shelvery_tests.conftest import destination_account - -print(f"Python lib path:\n{sys.path}") - -class ShelveryDocDBIntegrationTestCase(unittest.TestCase): - """Shelvery DocDB Backups Integration shelvery tests""" - def id(self): - return str(self.__class__) +print(f"Python lib path:\n{sys.path}") - def setUp(self): - self.created_snapshots = [] - self.regional_snapshots = { - 'ap-southeast-1': [], - 'ap-southeast-2': [] - } - os.environ['SHELVERY_MONO_THREAD'] = '1' - - # Complete initial setup and create service client - initSetup(self,'docdb') - docdbclient = AwsHelper.boto3_client('docdb', region_name='ap-southeast-2') - rdsclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') - - #Wait till db is ready - waiter = rdsclient.get_waiter('db_cluster_available') +class DocDBTestClass(ResourceClass): + + def __init__(self): + self.resource_name = DOCDB_RESOURCE_NAME + self.backups_engine = ShelveryDocumentDbBackup() + self.client = AwsHelper.boto3_client('docdb', region_name='ap-southeast-2') + self.ARN = f"arn:aws:rds:{os.environ['AWS_DEFAULT_REGION']}:{AwsHelper.local_account_id()}:cluster:{self.resource_name}" + + def add_backup_tags(self): + self.client.add_tags_to_resource( + ResourceName=self.ARN, + Tags=[{ + 'Key': f"{RuntimeConfig.get_tag_prefix()}:{ShelveryEngine.BACKUP_RESOURCE_TAG}", + 'Value': 'true' + }, + {'Key': 'Name', + 'Value': self.resource_name + } + ] + ) + + def wait_for_resource(self): + waiter = AwsHelper.boto3_client('rds', region_name='ap-southeast-2').get_waiter('db_cluster_available') try: waiter.wait( - DBClusterIdentifier='shelvery-test-docdb', + DBClusterIdentifier=self.resource_name, WaiterConfig={ 'Delay': 30, 'MaxAttempts': 50 @@ -66,90 +58,110 @@ def setUp(self): print(error) raise error - - #Get cluster name - clustername = f"arn:aws:rds:{os.environ['AWS_DEFAULT_REGION']}:{self.id['Account']}:cluster:shelvery-test-docdb" - - #Add tags to indicate backup - addBackupTags(docdbclient, - clustername, - "shelvery-test-docdb") +######## Test Case +class ShelveryDocDBIntegrationTestCase(unittest.TestCase): + """Shelvery DocDB Backups Integration shelvery tests""" + + def id(self): + return str(self.__class__) - self.share_with_id = destination_account + def setUp(self): + # Complete initial setup + self.created_snapshots = [] + setup_source(self) + # Instantiate resource test class + docdb_test_class = DocDBTestClass() + # Wait till DocDB Cluster is in an available state + docdb_test_class.wait_for_resource() + # Add tags to indicate backup + docdb_test_class.add_backup_tags() @pytest.mark.source - def test_Cleanup(self): - print(f"doc db - Running cleanup test") - docdb_backups_engine = ShelveryDocumentDbBackup() - backups = initCleanup(docdb_backups_engine) - docdb_client = AwsHelper.boto3_client('docdb') - - valid = False - for backup in backups: - valid = clusterCleanupBackups(self=self, - backup=backup, - backup_engine=docdb_backups_engine, - resource_client=docdb_client) - - valid = True + def test_CleanupDocDbBackup(self): + print(f"Doc DB - Running cleanup test") + # Create test resource class + docdb_test_class = DocDBTestClass() + backups_engine = docdb_test_class.backups_engine + client = docdb_test_class.client + # Create backups + backups = backups_engine.create_backups() + # Clean backups + backups_engine.clean_backups() + # Retrieve remaining backups + snapshots = [ + snapshot + for backup in backups + for snapshot in client.describe_db_cluster_snapshots( + DBClusterIdentifier=docdb_test_class.resource_name, + DBClusterSnapshotIdentifier=backup.backup_id + )["DBClusterSnapshots"] + ] + print(f"Snapshots: {snapshots}") - self.assertTrue(valid) + self.assertTrue(len(snapshots) == 0) @pytest.mark.source def test_CreateDocDbBackup(self): - - print(f"docdb - Running backup test") - - docdb_cluster_backup_engine = ShelveryDocumentDbBackup() - print(f"docdb - Shelvery backup initialised") + print("Running DocDB create backup test") + # Instantiate test resource class + docdb_test_class = DocDBTestClass() + backups_engine = docdb_test_class.backups_engine - backups = initCreateBackups(docdb_cluster_backup_engine) - print("Created Doc DB Cluster backups") - - valid = False + # Create backups + backups = backups_engine.create_backups() + print(f"Created {len(backups)} backups for Doc DB cluster") - # validate there is + # Compare backups for backup in backups: - valid = compareBackups(self=self, - backup=backup, - backup_engine=docdb_cluster_backup_engine - ) - self.assertTrue(valid) + valid = compare_backups(self=self, backup=backup, backup_engine=backups_engine) + + # Clean backups + print(f"Cleaning up DocDB Backups") + backups_engine.clean_backups() + + #Validate backup + self.assertTrue(valid, f"Backup {backup} is not valid") + + self.assertEqual(len(backups), 1, f"Expected 1 backup, but found {len(backups)}") @pytest.mark.source @pytest.mark.share def test_ShareDocDbBackup(self): - print(f"doc db - Running share backup test") - docdb_cluster_backup_engine = ShelveryDocumentDbBackup() + print("Running Doc DB share backup test") - print("Creating shared backups") - backups = initShareBackups(docdb_cluster_backup_engine, str(self.share_with_id)) + # Instantiate test resource class + docdb_test_class = DocDBTestClass() + backups_engine = docdb_test_class.backups_engine + client = docdb_test_class.client - print("Shared backups created") + print("Creating shared backups") + backups = backups_engine.create_backups() + print(f"{len(backups)} shared backups created") - valid = False - # validate there is for backup in backups: - valid = clusterShareBackups(self=self, - backup=backup, - service='docdb' + snapshot_id = backup.backup_id + print(f"Checking if snapshot {snapshot_id} is shared with {self.share_with_id}") + + # Retrieve snapshots + snapshots = client.describe_db_cluster_snapshots( + DBClusterIdentifier=docdb_test_class.resource_name, + DBClusterSnapshotIdentifier=backup.backup_id + )["DBClusterSnapshots"] + + # Get attributes of snapshot + attributes = client.describe_db_cluster_snapshot_attributes( + DBClusterSnapshotIdentifier=snapshot_id + )['DBClusterSnapshotAttributesResult']['DBClusterSnapshotAttributes'] + + # Check if snapshot is shared with destination account + shared_with_destination = any( + attr['AttributeName'] == 'restore' and self.share_with_id in attr['AttributeValues'] + for attr in attributes ) - + + # Assertions + self.assertEqual(len(snapshots), 1, f"Expected 1 snapshot, but found {len(snapshots)}") + self.assertTrue(shared_with_destination, f"Snapshot {snapshot_id} is not shared with {self.share_with_id}") - - self.assertTrue(valid) - - def tearDown(self): - print("doc db - tear down doc db snapshot") - docdbclient = AwsHelper.boto3_client('docdb', region_name='ap-southeast-2') - for snapid in self.created_snapshots: - print(f"Deleting snapshot {snapid}") - try: - docdbclient.delete_db_cluster_snapshot(DBClusterSnapshotIdentifier=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - - print("docdb - snapshot deleted, instance deleting") - if __name__ == '__main__': unittest.main() diff --git a/shelvery_tests/docdb_pull_test.py b/shelvery_tests/docdb_pull_test.py index 3b18ef9..62919b0 100644 --- a/shelvery_tests/docdb_pull_test.py +++ b/shelvery_tests/docdb_pull_test.py @@ -1,16 +1,11 @@ import sys -import traceback import unittest -import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime +import pytest +from shelvery_tests.docdb_integration_test import DocDBTestClass +from shelvery_tests.test_functions import setup_destination +from shelvery_tests.resources import DOCDB_RESOURCE_NAME -from shelvery_tests.test_functions import addBackupTags pwd = os.path.dirname(os.path.abspath(__file__)) @@ -20,51 +15,41 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.documentdb_backup import ShelveryDocumentDbBackup -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper -from shelvery_tests.conftest import source_account -from shelvery_tests.cleanup_functions import cleanDocDBSnapshots - -#Need to add 'source acc' to env -#Call create_data_bucket -#How to cleanup source account after running in dest? - class ShelveryDocDBPullTestCase(unittest.TestCase): @pytest.mark.destination def test_PullDocDbBackup(self): - os.environ['SHELVERY_MONO_THREAD'] = '1' - - cleanDocDBSnapshots() - - source_aws_id = source_account - os.environ["shelvery_source_aws_account_ids"] = str(source_aws_id) - print(f"doc db - Running pull shared backups test") + # Complete initial setup + print(f"Doc DB - Running pull shared backups test") + setup_destination(self) - docdbclient = AwsHelper.boto3_client('docdb', region_name='ap-southeast-2') - docdb_cluster_backup_engine = ShelveryDocumentDbBackup() - - print("Pulling shared backups") - docdb_cluster_backup_engine.pull_shared_backups() + # Instantiate test resource class + docdb_test_class = DocDBTestClass() + backups_engine = docdb_test_class.backups_engine + client = docdb_test_class.client + + # Clean residual existing snapshots + backups_engine.clean_backups() - #Get post-pull snapshot count - pulled_snapshot = docdbclient.describe_db_cluster_snapshots( - DBClusterIdentifier='shelvery-test-docdb', + # Pull shared backups + backups_engine.pull_shared_backups() + + # Get post-pull snapshot count + pulled_snapshots = client.describe_db_cluster_snapshots( + DBClusterIdentifier=DOCDB_RESOURCE_NAME, SnapshotType='Manual' ) - - print("PULLED:" + str(pulled_snapshot)) - self.assertTrue(len(pulled_snapshot['DBClusterSnapshots']) == 1) + # Verify that only one snapshot was pulled + self.assertEqual(len(pulled_snapshots['DBClusterSnapshots']), 1) @pytest.mark.cleanup def test_cleanup(self): - cleanDocDBSnapshots() - + # Instantiate test resource class + docdb_test_class = DocDBTestClass() + backups_engine = docdb_test_class.backups_engine + # Clean backups + backups_engine.clean_backups() \ No newline at end of file diff --git a/shelvery_tests/ebs_integration_test.py b/shelvery_tests/ebs_integration_test.py index cd108b2..7b3c7a1 100644 --- a/shelvery_tests/ebs_integration_test.py +++ b/shelvery_tests/ebs_integration_test.py @@ -1,17 +1,15 @@ import sys -import traceback import unittest import pytest -import yaml - -import boto3 import os import time -import botocore -from datetime import datetime +from botocore.exceptions import WaiterError +from shelvery.engine import ShelveryEngine +from shelvery.runtime_config import RuntimeConfig +from shelvery_tests.test_functions import setup_source, compare_backups from shelvery.ebs_backup import ShelveryEBSBackup - -from shelvery_tests.test_functions import compareBackups, createBackupTags, ebsCleanupBackups, ec2ShareBackups, initCleanup, initCreateBackups, initSetup, initShareBackups +from shelvery.aws_helper import AwsHelper +from shelvery_tests.resources import EBS_INSTANCE_RESOURCE_NAME, ResourceClass pwd = os.path.dirname(os.path.abspath(__file__)) @@ -21,15 +19,54 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper -from shelvery_tests.conftest import destination_account - print(f"Python lib path:\n{sys.path}") +class EBSTestClass(ResourceClass): + + def __init__(self): + self.resource_name = EBS_INSTANCE_RESOURCE_NAME + self.backups_engine = ShelveryEBSBackup() + self.client = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2') + self.resource_id = self.get_instance_id() + + def add_backup_tags(self): + self.client.create_tags( + Resources=[self.resource_id], + Tags=[{ + 'Key': f"{RuntimeConfig.get_tag_prefix()}:{ShelveryEngine.BACKUP_RESOURCE_TAG}", + 'Value': 'true' + }, + {'Key': 'Name', + 'Value': self.resource_name + }] + ) + + def get_instance_id(self): + # Find EBS Volume + search_filter = [{'Name': 'tag:Name', 'Values': [self.resource_name]}] + # Get EBS volumes + ebs_volumes = self.client.describe_volumes(Filters=search_filter) + # Get volume ID + try: + return ebs_volumes['Volumes'][0]['VolumeId'] + except (IndexError, KeyError): + print("No EBS volumes found matching the given criteria.") + return "" + + def wait_for_resource(self): + waiter = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2').get_waiter('volume_available') + try: + waiter.wait( + VolumeIds=[self.resource_id], + WaiterConfig={ + 'Delay': 30, + 'MaxAttempts': 50 + } + ) + except WaiterError as error: + print("Waiting for EBS Volume Failed") + print(error) + raise error class ShelveryEBSIntegrationTestCase(unittest.TestCase): """Shelvery EBS Backups Integration shelvery tests""" @@ -38,127 +75,116 @@ def id(self): return str(self.__class__) def setUp(self): + # Complete initial setup self.created_snapshots = [] - self.regional_snapshots = { - 'ap-southeast-1': [], - 'ap-southeast-2': [] - } - os.environ['SHELVERY_MONO_THREAD'] = '1' - - # Create and configure RDS artefact - initSetup(self,'ec2') - ec2client = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2') - - #Find ec2 instance - search_filter = [{'Name':'tag:Name', - 'Values': ['shelvery-test-ebs'] - }] - - #Get ebs volume - ebs_volume = ec2client.describe_volumes(Filters = search_filter) - - print("Volumes: " + str(ebs_volume)) + self.regional_snapshots = [] + setup_source(self) + # Instantiate resource test class + ebs_test_class = EBSTestClass() + # Wait till volume is in an available state + ebs_test_class.wait_for_resource() + # Add tags to indicate backup + ebs_test_class.add_backup_tags() - volume_id = ebs_volume['Volumes'][0]['VolumeId'] - - createBackupTags(ec2client,[volume_id],"shelvery-test-ebs") - - self.share_with_id = destination_account @pytest.mark.source - def test_Cleanup(self): - print(f"ebs - Running cleanup test") - ebs_backup_engine = ShelveryEBSBackup() - backups = initCleanup(ebs_backup_engine) - ec2_client = AwsHelper.boto3_client('ec2') - - valid = False - for backup in backups: - valid = ebsCleanupBackups(self=self, - backup=backup, - backup_engine=ebs_backup_engine, - service_client=ec2_client) + def test_CleanupEbsBackup(self): + print(f"EBS - Running cleanup test") + # Create test resource class + ebs_test_class = EBSTestClass() + backups_engine = ebs_test_class.backups_engine + client = ebs_test_class.client + # Create backups + backups = backups_engine.create_backups() + # Clean backups + backups_engine.clean_backups() + # Retrieve remaining backups + snapshots = [ + snapshot + for backup in backups + for snapshot in client.describe_snapshots( + Filters = [{ + 'Name': 'tag:Name', + 'Values': [backup.name] + }] + )['Snapshots'] + ] + print(f"Snapshots: {snapshots}") - self.assertTrue(valid) + self.assertTrue(len(snapshots) == 0) + @pytest.mark.source def test_CreateEbsBackup(self): - print(f"ebs - Running backup test") - - ebs_backup_engine = ShelveryEBSBackup() - print(f"ebs - Shelvery backup initialised") - - backups = initCreateBackups(ebs_backup_engine) - - ec2_client = AwsHelper.boto3_client('ec2') - - valid = False - # validate there is + print("Running EBS create backup test") + # Create test resource class + ebs_test_class = EBSTestClass() + backups_engine = ebs_test_class.backups_engine + + # Create backups + backups = backups_engine.create_backups() + print(f"Created {len(backups)} backups for EBS Volume") + + # Compare backups for backup in backups: - - #Get source snapshot - source_snapshot = ec2_client.describe_snapshots( - Filters = [{ - 'Name': 'tag:Name', - 'Values': [ - backup.name - ] - }] - ) - - #Get snapshot id and add to created snapshot list for removal in teardown later - dest_snapshot_id = source_snapshot['Snapshots'][0]['SnapshotId'] - self.created_snapshots.append(dest_snapshot_id) - - valid = compareBackups(self=self, - backup=backup, - backup_engine=ebs_backup_engine - ) - - self.assertTrue(valid) - + valid = compare_backups(self=self, backup=backup, backup_engine=backups_engine) + + # Clean backups + print(f"Cleaning up EBS Backups") + backups_engine.clean_backups() + + # Validate Backup + self.assertTrue(valid, f"Backup {backup} is not valid") + + self.assertEqual(len(backups), 1, f"Expected 1 backup, but found {len(backups)}") + @pytest.mark.source @pytest.mark.share def test_ShareEbsBackup(self): - print(f"ebs - Running share backup test") - ebs_backup_engine = ShelveryEBSBackup() + print("Running EBS share backup test") + # Instantiate test resource classs + ebs_test_class = EBSTestClass() + backups_engine = ebs_test_class.backups_engine + client = ebs_test_class.client print("Creating shared backups") - backups = initShareBackups(ebs_backup_engine, str(self.share_with_id)) + backups = backups_engine.create_backups() + print(f"{len(backups)} shared backups created") - print("Shared backups created") - - valid = False - # validate there is for backup in backups: - valid = ec2ShareBackups(self=self, - backup=backup - ) - self.assertTrue(valid) - - def tearDown(self): - ec2client = AwsHelper.boto3_client('ec2') - time.sleep(20) - # snapshot deletion surrounded with try/except in order - # for cases when shelvery cleans / does not clean up behind itself - for snapid in self.created_snapshots: - - print(f"Deleting snapshot {snapid}") - try: - ec2client.delete_snapshot(SnapshotId=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - - for region in self.regional_snapshots: - ec2regional = AwsHelper.boto3_client('ec2', region_name=region) - for snapid in self.regional_snapshots[region]: - try: - ec2regional.delete_snapshot(SnapshotId=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - + snapshot_id = backup.backup_id + print(f"Checking if snapshot {snapshot_id} is shared with {self.share_with_id}") + + # Retrieve snapshot + snapshots = client.describe_snapshots( + Filters = [{ + 'Name': 'tag:Name', + 'Values': [backup.name] + }] + )['Snapshots'] + + snapshot_id = snapshots[0]['SnapshotId'] + + # retrieve the snapshot attributes + response = client.describe_snapshot_attribute( + SnapshotId=snapshot_id, + Attribute='createVolumePermission' + ) + + # check if the snapshot is shared with the destination account + shared_with_destination = any( + perm['UserId'] == self.share_with_id for perm in response.get('CreateVolumePermissions', []) + ) + + # Assertions + self.assertEqual(len(snapshots), 1, f"Expected 1 snapshot, but found {len(snapshots)}") + self.assertTrue(shared_with_destination, f"Snapshot {snapshot_id} is not shared with {self.share_with_id}") + def tearDown(self): + print("Waiting 30s due to EBS Snapshot rate limit...") + time.sleep(30) #EBS snapshot create wait limit, so wait 30~ + if __name__ == '__main__': unittest.main() diff --git a/shelvery_tests/ebs_pull_test.py b/shelvery_tests/ebs_pull_test.py index 6cdfc78..3dfc856 100644 --- a/shelvery_tests/ebs_pull_test.py +++ b/shelvery_tests/ebs_pull_test.py @@ -1,16 +1,11 @@ import sys -import traceback import unittest -import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime +import pytest +from shelvery_tests.ebs_integration_test import EBSTestClass +from shelvery_tests.test_functions import setup_destination +from shelvery_tests.resources import EBS_INSTANCE_RESOURCE_NAME -from shelvery_tests.test_functions import cleanEC2Snapshots, compareBackups, createBackupTags, ebsCleanupBackups, ebsPullBackups, initCleanup, initCreateBackups, initSetup, initShareBackups pwd = os.path.dirname(os.path.abspath(__file__)) @@ -20,21 +15,43 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.ebs_backup import ShelveryEBSBackup -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper - - class ShelveryEBSPullTestCase(unittest.TestCase): @pytest.mark.destination def test_PullEBSBackup(self): - os.environ['SHELVERY_MONO_THREAD'] = '1' - ebs_client = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2') - ebs_backup_engine = ShelveryEBSBackup() - - ebsPullBackups(self,ebs_client,ebs_backup_engine,'shelvery-test-ebs') + # Complete initial setup + print(f"EBS - Running pull shared backups test") + setup_destination(self) + + # Create test resource class + ebs_test_class = EBSTestClass() + backups_engine = ebs_test_class.backups_engine + client = ebs_test_class.client + + # Clean residual existing snapshots + backups_engine.clean_backups() + + # Pull shared backups + backups_engine.pull_shared_backups() + + # Get post-pull snapshot count + search_filter = [{'Name':'tag:ResourceName', + 'Values':[EBS_INSTANCE_RESOURCE_NAME] + }] + + #Retrieve pulled images from shelvery-test stack + snapshots = client.describe_snapshots( + Filters=search_filter + )['Snapshots'] + + # Verify that only one snapshot was pulled + self.assertEqual(len(snapshots), 1) + + @pytest.mark.cleanup + def test_cleanup(self): + # Create test resource class + ebs_test_class = EBSTestClass() + backups_engine = ebs_test_class.backups_engine + # Clean backups + backups_engine.clean_backups() diff --git a/shelvery_tests/ec2ami_integration_test.py b/shelvery_tests/ec2ami_integration_test.py index afc6374..2a5f8ba 100644 --- a/shelvery_tests/ec2ami_integration_test.py +++ b/shelvery_tests/ec2ami_integration_test.py @@ -1,16 +1,15 @@ import sys -import traceback import unittest import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime +from botocore.exceptions import WaiterError +from shelvery.engine import ShelveryEngine +from shelvery.runtime_config import RuntimeConfig +from shelvery_tests.test_functions import setup_source, compare_backups +from shelvery.ec2ami_backup import ShelveryEC2AMIBackup +from shelvery.aws_helper import AwsHelper +from shelvery_tests.resources import EC2_AMI_INSTANCE_RESOURCE_NAME, ResourceClass -from shelvery_tests.test_functions import compareBackups, createBackupTags, ec2CleanupBackups, ec2ShareBackups, initCleanup, initCreateBackups, initSetup, initShareBackups pwd = os.path.dirname(os.path.abspath(__file__)) @@ -20,18 +19,60 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.ec2ami_backup import ShelveryEC2AMIBackup -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper -from shelvery_tests.conftest import destination_account - print(f"Python lib path:\n{sys.path}") - -## Add check for non-terminated ec2am? (because it takes ages to transition from terminated -> deleted) +import boto3 +class EC2AmiTestClass(ResourceClass): + + def __init__(self): + self.resource_name = EC2_AMI_INSTANCE_RESOURCE_NAME + self.backups_engine = ShelveryEC2AMIBackup() + self.client = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2') + self.resource_id = self.get_instance_id() + + def add_backup_tags(self): + self.client.create_tags( + Resources=[self.resource_id], + Tags=[{ + 'Key': f"{RuntimeConfig.get_tag_prefix()}:{ShelveryEngine.BACKUP_RESOURCE_TAG}", + 'Value': 'true' + }, + {'Key': 'Name', + 'Value': self.resource_name + }] + ) + + def get_instance_id(self): + # Find EC2 instance + search_filter = [ + {'Name': 'tag:Name', 'Values': [EC2_AMI_INSTANCE_RESOURCE_NAME]}, + {'Name': 'instance-state-name', 'Values': ['running']} + ] + + # Get EC2 instance + ec2_instance = self.client.describe_instances(Filters=search_filter) + + # Get instance ID + try: + return ec2_instance['Reservations'][0]['Instances'][0]['InstanceId'] + except (IndexError, KeyError): + print("No instance found matching the given criteria.") + return "" + + def wait_for_resource(self): + waiter = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2').get_waiter('instance_running') + try: + waiter.wait( + InstanceIds=[self.resource_id], + WaiterConfig={ + 'Delay': 30, + 'MaxAttempts': 50 + } + ) + except WaiterError as error: + print("Waiting for EC2 Instance Failed") + print(error) + raise error class ShelveryEC2AmiIntegrationTestCase(unittest.TestCase): """Shelvery EC2 AMI Backups Integration shelvery tests""" @@ -40,146 +81,109 @@ def id(self): return str(self.__class__) def setUp(self): + # Complete initial setup self.created_snapshots = [] - self.regional_snapshots = { - 'ap-southeast-1': [], - 'ap-southeast-2': [] - } - os.environ['SHELVERY_MONO_THREAD'] = '1' - - # Create and configure RDS artefact - initSetup(self,'ec2') - ec2client = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2') - - #Find ec2 instance - search_filter = [{'Name':'tag:Name', - 'Values': ['shelvery-test-ec2'], - 'Name': 'instance-state-name', - 'Values': ['running'] - }] - - - #Get ec2 instance - ec2_instance = ec2client.describe_instances(Filters = search_filter) - - #Get instance id - instance_id = ec2_instance['Reservations'][0]['Instances'][0]['InstanceId'] - print("INSTANCE ID: " + str(instance_id)) - - createBackupTags(ec2client,[instance_id],"shelvery-test-ec2") - - self.share_with_id = destination_account + self.regional_snapshots = [] + setup_source(self) + # Instantiate resource test class + ec2_ami_test_class = EC2AmiTestClass() + # Wait till instance is in an available state + ec2_ami_test_class.wait_for_resource() + # Add tags to indicate backup + ec2_ami_test_class.add_backup_tags() @pytest.mark.source - def test_Cleanup(self): - print(f"ec2 ami - Running cleanup test") - ec2_ami_backup_engine = ShelveryEC2AMIBackup() - backups = initCleanup(ec2_ami_backup_engine) - ec2_client = AwsHelper.boto3_client('ec2') - - valid = False - for backup in backups: - valid = ec2CleanupBackups(self=self, - backup=backup, - backup_engine=ec2_ami_backup_engine, - service_client=ec2_client) + def test_CleanupEC2AmiBackup(self): + + print(f"EC2 Ami - Running cleanup test") + # Create test resource class + ec2_ami_test_class = EC2AmiTestClass() + backups_engine = ec2_ami_test_class.backups_engine + client = ec2_ami_test_class.client + # Create backups + backups = backups_engine.create_backups() + # Clean backups + backups_engine.clean_backups() + # Retrieve remaining backups + snapshots = [ + snapshot + for backup in backups + for snapshot in client.describe_snapshots( + Filters = [{ + 'Name': 'tag:Name', + 'Values': [backup.name] + }] + )['Snapshots'] + ] + print(f"Snapshots: {snapshots}") - self.assertTrue(valid) + self.assertTrue(len(snapshots) == 0) @pytest.mark.source def test_CreateEc2AmiBackup(self): - print(f"ec2 ami - Running backup test") - - ec2_ami_backup_engine = ShelveryEC2AMIBackup() - print(f"ec2 ami - Shelvery backup initialised") - - backups = initCreateBackups(ec2_ami_backup_engine) - - ec2_client = AwsHelper.boto3_client('ec2') - - valid = False - # validate there is + print("Running EC2 AMI create backup test") + # Create test resource class + ec2_ami_test_class = EC2AmiTestClass() + backups_engine = ec2_ami_test_class.backups_engine + + # Create backups + backups = backups_engine.create_backups() + print(f"Created {len(backups)} backups for EC2 Instance") + + # Compare backups for backup in backups: - - #Get source snapshot - source_snapshot = ec2_client.describe_snapshots( - Filters = [{ - 'Name': 'tag:Name', - 'Values': [ - backup.name - ] - }] - ) - - #Get snapshot id and add to created snapshot list for removal in teardown later - dest_snapshot_id = source_snapshot['Snapshots'][0]['SnapshotId'] - self.created_snapshots.append(dest_snapshot_id) - - valid = compareBackups(self=self, - backup=backup, - backup_engine=ec2_ami_backup_engine - ) - - - self.assertTrue(valid) - + valid = compare_backups(self=self, backup=backup, backup_engine=backups_engine) + + # Clean backups + print(f"Cleaning up EC2 AMI Backups") + backups_engine.clean_backups() + + #Validate backup + self.assertTrue(valid, f"Backup {backup} is not valid") + + self.assertEqual(len(backups), 1, f"Expected 1 backup, but found {len(backups)}") + @pytest.mark.source @pytest.mark.share def test_ShareEc2AmiBackup(self): - - print(f"ec2 ami - Running share backup test") - ec2_ami_backup_engine = ShelveryEC2AMIBackup() + print("Running EC2 AMI share backup test") + # Instantiate test resource classs + ec2_ami_test_class = EC2AmiTestClass() + backups_engine = ec2_ami_test_class.backups_engine + client = boto3.client('ec2')#ec2_ami_test_class.client print("Creating shared backups") - backups = initShareBackups(ec2_ami_backup_engine, str(self.share_with_id)) - - print("Shared backups created") + backups = backups_engine.create_backups() + print(f"{len(backups)} shared backups created") - valid = False - # validate there is for backup in backups: - valid = ec2ShareBackups(self=self, - backup=backup, - ) - self.assertTrue(valid) - - def tearDown(self): - ec2client = AwsHelper.boto3_client('ec2') - # snapshot deletion surrounded with try/except in order - # for cases when shelvery cleans / does not clean up behind itself - time.sleep(20) - for snapid in self.created_snapshots: - print(f"Deleting snapshot {snapid}") - try: - snapshot = ec2client.describe_snapshots( - SnapshotIds = [snapid] - ) - - print(snapshot) - - tags = snapshot['Snapshots'][0]['Tags'] - - print("TAGS") - print(tags) - - ami_id = [tag['Value'] for tag in tags if tag['Key'] == 'shelvery:ami_id'][0] - - print("AMI") - print(ami_id) - - ec2client.deregister_image(ImageId=ami_id) - ec2client.delete_snapshot(SnapshotId=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - - for region in self.regional_snapshots: - ec2regional = AwsHelper.boto3_client('ec2', region_name=region) - for snapid in self.regional_snapshots[region]: - try: - ec2regional.delete_snapshot(SnapshotId=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - + snapshot_id = backup.backup_id + print(f"Checking if snapshot {snapshot_id} is shared with {self.share_with_id}") + + # Retrieve snapshot + snapshots = client.describe_snapshots( + Filters = [{ + 'Name': 'tag:Name', + 'Values': [backup.name] + }] + )['Snapshots'] + + snapshot_id = snapshots[0]['SnapshotId'] + + # retrieve the snapshot attributes + response = client.describe_snapshot_attribute( + SnapshotId=snapshot_id, + Attribute='createVolumePermission' + ) + + # check if the snapshot is shared with the destination account + shared_with_destination = any( + perm['UserId'] == self.share_with_id for perm in response.get('CreateVolumePermissions', []) + ) + + # Assertions + self.assertEqual(len(snapshots), 1, f"Expected 1 snapshot, but found {len(snapshots)}") + self.assertTrue(shared_with_destination, f"Snapshot {snapshot_id} is not shared with {self.share_with_id}") if __name__ == '__main__': diff --git a/shelvery_tests/ec2ami_pull_test.py b/shelvery_tests/ec2ami_pull_test.py index a4fd0ea..5261bfe 100644 --- a/shelvery_tests/ec2ami_pull_test.py +++ b/shelvery_tests/ec2ami_pull_test.py @@ -1,17 +1,10 @@ import sys -import traceback import unittest -import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime - -from shelvery_tests.test_functions import cleanEC2Snapshots, compareBackups, createBackupTags, ec2CleanupBackups, ec2PullBackups, ec2ShareBackups, initCleanup, initCreateBackups, initSetup, initShareBackups - +import pytest +from shelvery_tests.ec2ami_integration_test import EC2AmiTestClass +from shelvery_tests.test_functions import setup_destination +from shelvery_tests.resources import EC2_AMI_INSTANCE_RESOURCE_NAME pwd = os.path.dirname(os.path.abspath(__file__)) sys.path.append(f"{pwd}/..") @@ -20,26 +13,42 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.ec2ami_backup import ShelveryEC2AMIBackup -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper - - class ShelveryEC2AmiPullTestCase(unittest.TestCase): @pytest.mark.destination def test_PullEC2Backup(self): - os.environ['SHELVERY_MONO_THREAD'] = '1' - print(f"ec2 - Running pull shared backups test") + # Complete initial setup + print(f"EC2 AMI - Running pull shared backups test") + setup_destination(self) - ec2_client = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2') - ec2_backup_engine = ShelveryEC2AMIBackup() - - ec2PullBackups(self,ec2_client,ec2_backup_engine) + # Create test resource class + ec2_ami_test_class = EC2AmiTestClass() + backups_engine = ec2_ami_test_class.backups_engine + client = ec2_ami_test_class.client + + # Clean residual existing snapshots + backups_engine.clean_backups() + + # Pull shared backups + backups_engine.pull_shared_backups() + + # Get post-pull snapshot count + search_filter = [{'Name':'tag:ResourceName', + 'Values':[EC2_AMI_INSTANCE_RESOURCE_NAME] + }] + + #Retrieve pulled images from shelvery-test stack + amis = client.describe_images( + Filters=search_filter + )["Images"] + + # Verify that only one snapshot was pulled + self.assertEqual(len(amis), 1) @pytest.mark.cleanup def test_cleanup(self): - cleanEC2Snapshots() + # Create test resource class + ec2_ami_test_class = EC2AmiTestClass() + backups_engine = ec2_ami_test_class.backups_engine + # Clean backups + backups_engine.clean_backups() diff --git a/shelvery_tests/rds_cluster_integration_test.py b/shelvery_tests/rds_cluster_integration_test.py index 8194e46..7c2671f 100644 --- a/shelvery_tests/rds_cluster_integration_test.py +++ b/shelvery_tests/rds_cluster_integration_test.py @@ -1,17 +1,14 @@ import sys -import traceback -from turtle import back import unittest import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime - -from shelvery_tests.test_functions import clusterCleanupBackups, clusterShareBackups, compareBackups, initCleanup, initCreateBackups, addBackupTags, initShareBackups, initSetup +from botocore.exceptions import WaiterError +from shelvery.engine import ShelveryEngine +from shelvery.runtime_config import RuntimeConfig +from shelvery_tests.test_functions import setup_source, compare_backups +from shelvery.rds_cluster_backup import ShelveryRDSClusterBackup +from shelvery.aws_helper import AwsHelper +from shelvery_tests.resources import RDS_CLUSTER_RESOURCE_NAME, ResourceClass pwd = os.path.dirname(os.path.abspath(__file__)) @@ -21,17 +18,44 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.rds_cluster_backup import ShelveryRDSClusterBackup -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper -from shelvery_tests.conftest import destination_account - - print(f"Python lib path:\n{sys.path}") +class RDSClusterTestClass(ResourceClass): + + def __init__(self): + self.resource_name = RDS_CLUSTER_RESOURCE_NAME + self.backups_engine = ShelveryRDSClusterBackup() + self.client = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') + self.ARN = f"arn:aws:rds:{os.environ['AWS_DEFAULT_REGION']}:{AwsHelper.local_account_id()}:cluster:{self.resource_name}" + + def add_backup_tags(self): + self.client.add_tags_to_resource( + ResourceName=self.ARN, + Tags=[{ + 'Key': f"{RuntimeConfig.get_tag_prefix()}:{ShelveryEngine.BACKUP_RESOURCE_TAG}", + 'Value': 'true' + }, + {'Key': 'Name', + 'Value': self.resource_name + } + ] + ) + + def wait_for_resource(self): + waiter = AwsHelper.boto3_client('rds', region_name='ap-southeast-2').get_waiter('db_cluster_available') + try: + waiter.wait( + DBClusterIdentifier=self.resource_name, + WaiterConfig={ + 'Delay': 30, + 'MaxAttempts': 50 + } + ) + except WaiterError as error: + print("Waiting for RDS Cluster Failed") + print(error) + raise error + class ShelveryRDSClusterIntegrationTestCase(unittest.TestCase): """Shelvery RDS Cluster Backups Integration shelvery tests""" @@ -41,99 +65,102 @@ def id(self): def setUp(self): + # Complete initial setup self.created_snapshots = [] - self.regional_snapshots = { - 'ap-southeast-1': [], - 'ap-southeast-2': [] - } - os.environ['SHELVERY_MONO_THREAD'] = '1' - - # Complete initial setup and create service client - initSetup(self,'rds') - rdsclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') - - - #Get cluster name - clustername = f"arn:aws:rds:{os.environ['AWS_DEFAULT_REGION']}:{self.id['Account']}:cluster:shelvery-test-rds-cluster" - - #Add tags to indicate backup - addBackupTags(rdsclient, - clustername, - "shelvery-test-rds-cluster") - - self.share_with_id = destination_account + setup_source(self) + # Instantiate resource test class + rds_cluster_test_class = RDSClusterTestClass() + # Wait till RDS Cluster is in an available state + rds_cluster_test_class.wait_for_resource() + # Add tags to indicate backup + rds_cluster_test_class.add_backup_tags() @pytest.mark.source - def test_Cleanup(self): - print(f"rds cluster - Running cleanup test") - rdscluster_backups_engine = ShelveryRDSClusterBackup() - backups = initCleanup(rdscluster_backups_engine) - rdscluster_client = AwsHelper.boto3_client('rds') - - valid = False - for backup in backups: - valid = clusterCleanupBackups(self=self, - backup=backup, - backup_engine=rdscluster_backups_engine, - resource_client=rdscluster_client) + def test_CleanupRdsClusterBackup(self): + print(f"RDS Cluster - Running cleanup test") + # Create test resource class + rds_cluster_test_class = RDSClusterTestClass() + backups_engine = rds_cluster_test_class.backups_engine + client = rds_cluster_test_class.client + # Create backups + backups = backups_engine.create_backups() + # Clean backups + backups_engine.clean_backups() + # Retrieve remaining backups + snapshots = [ + snapshot + for backup in backups + for snapshot in client.describe_db_cluster_snapshots( + DBClusterIdentifier=rds_cluster_test_class.resource_name, + DBClusterSnapshotIdentifier=backup.backup_id + )["DBClusterSnapshots"] + ] + print(f"Snapshots: {snapshots}") - self.assertTrue(valid) + self.assertTrue(len(snapshots) == 0) @pytest.mark.source def test_CreateRdsClusterBackup(self): - - print(f"rds cluster - Running backup test") - - rds_cluster_backup_engine = ShelveryRDSClusterBackup() - print(f"rds cluster - Shelvery backup initialised") + print("Running RDS Cluster create backup test") + # Instantiate test resource class + rds_cluster_test_class = RDSClusterTestClass() + backups_engine = rds_cluster_test_class.backups_engine - backups = initCreateBackups(rds_cluster_backup_engine) - print("Created RDS Cluster backups") - - valid = False - - # validate there is + # Create backups + backups = backups_engine.create_backups() + print(f"Created {len(backups)} backups for RDS Cluster") + + # Compare backups for backup in backups: - valid = compareBackups(self=self, - backup=backup, - backup_engine=rds_cluster_backup_engine - ) - self.assertTrue(valid) - + valid = compare_backups(self=self, backup=backup, backup_engine=backups_engine) + + # Clean backups + print(f"Cleaning up RDS Cluster Backups") + backups_engine.clean_backups() + + # Validate backups + self.assertTrue(valid, f"Backup {backup} is not valid") + + self.assertEqual(len(backups), 1, f"Expected 1 backup, but found {len(backups)}") + @pytest.mark.source @pytest.mark.share def test_ShareRdsClusterBackup(self): + print("Running RDS Cluster share backup test") - print(f"rds cluster - Running share backup test") - rds_cluster_backup_engine = ShelveryRDSClusterBackup() + # Instantiate test resource class + rds_cluster_test_class = RDSClusterTestClass() + backups_engine = rds_cluster_test_class.backups_engine + client = rds_cluster_test_class.client print("Creating shared backups") - backups = initShareBackups(rds_cluster_backup_engine, str(self.share_with_id)) + backups = backups_engine.create_backups() + print(f"{len(backups)} shared backups created") - print("Shared backups created") - - valid = False - # validate there is for backup in backups: - valid = clusterShareBackups(self=self, - backup=backup, - service='rds' - ) - - self.assertTrue(valid) - - def tearDown(self): - print("rds cluster - tear down rds cluster snapshot") - rdsclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') - for snapid in self.created_snapshots: - print(f"Deleting snapshot {snapid}") - try: - rdsclient.delete_db_cluster_snapshot(DBClusterSnapshotIdentifier=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - - print("rds - snapshot deleted, instance deleting") - + snapshot_id = backup.backup_id + print(f"Checking if snapshot {snapshot_id} is shared with {self.share_with_id}") + + # Retrieve snapshots + snapshots = client.describe_db_cluster_snapshots( + DBClusterIdentifier=rds_cluster_test_class.resource_name, + DBClusterSnapshotIdentifier=backup.backup_id + )["DBClusterSnapshots"] + + # Get attributes of snapshot + attributes = client.describe_db_cluster_snapshot_attributes( + DBClusterSnapshotIdentifier=snapshot_id + )['DBClusterSnapshotAttributesResult']['DBClusterSnapshotAttributes'] + + # Check if snapshot is shared with destination account + shared_with_destination = any( + attr['AttributeName'] == 'restore' and self.share_with_id in attr['AttributeValues'] + for attr in attributes + ) + + # Assertions + self.assertEqual(len(snapshots), 1, f"Expected 1 snapshot, but found {len(snapshots)}") + self.assertTrue(shared_with_destination, f"Snapshot {snapshot_id} is not shared with {self.share_with_id}") if __name__ == '__main__': diff --git a/shelvery_tests/rds_cluster_pull_test.py b/shelvery_tests/rds_cluster_pull_test.py index c05f4bd..af2dd2f 100644 --- a/shelvery_tests/rds_cluster_pull_test.py +++ b/shelvery_tests/rds_cluster_pull_test.py @@ -1,15 +1,10 @@ import sys -import traceback import unittest -import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime -from shelvery.rds_cluster_backup import ShelveryRDSClusterBackup +import pytest +from shelvery_tests.rds_cluster_integration_test import RDSClusterTestClass +from shelvery_tests.test_functions import setup_destination +from shelvery_tests.resources import RDS_CLUSTER_RESOURCE_NAME pwd = os.path.dirname(os.path.abspath(__file__)) @@ -19,51 +14,43 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper -from shelvery_tests.conftest import source_account -from shelvery_tests.cleanup_functions import cleanRdsClusterSnapshots - -#Need to add 'source acc' to env -#Call create_data_bucket -#How to cleanup source account after running in dest? - class ShelveryRDSClusterPullTestCase(unittest.TestCase): @pytest.mark.destination def test_PullRdsClusterBackup(self): - os.environ['SHELVERY_MONO_THREAD'] = '1' - cleanRdsClusterSnapshots() - - source_aws_id = source_account - os.environ["shelvery_source_aws_account_ids"] = str(source_aws_id) - - print(f"rds cluster - Running pull shared backups test") + # Complete initial setup + print(f"RDS Cluster - Running pull shared backups test") + setup_destination(self) - rds_cluster_client = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') - rds_cluster_backup_engine = ShelveryRDSClusterBackup() - + # Instantiate test resource class + rds_cluster_test_class = RDSClusterTestClass() + backups_engine = rds_cluster_test_class.backups_engine + client = rds_cluster_test_class.client - print("Pulling shared backups") - rds_cluster_backup_engine.pull_shared_backups() + # Clean residual existing snapshots + backups_engine.clean_backups() - #Get post-pull snapshot count - pulled_snapshot = rds_cluster_client.describe_db_cluster_snapshots( - DBClusterIdentifier='shelvery-test-rds-cluster', + # Pull shared backups + backups_engine.pull_shared_backups() + + # Get post-pull snapshot count + pulled_snapshots = client.describe_db_cluster_snapshots( + DBClusterIdentifier=RDS_CLUSTER_RESOURCE_NAME, SnapshotType='Manual' ) - - print("PULLED:" + str(pulled_snapshot)) - self.assertTrue(len(pulled_snapshot['DBClusterSnapshots']) == 1) + # Verify that only one snapshot was pulled + self.assertEqual(len(pulled_snapshots['DBClusterSnapshots']), 1) @pytest.mark.cleanup def test_cleanup(self): - cleanRdsClusterSnapshots() + # Instantiate test resource class + rds_cluster_test_class = RDSClusterTestClass() + backups_engine = rds_cluster_test_class.backups_engine + # Clean backups + backups_engine.clean_backups() + diff --git a/shelvery_tests/rds_integration_test.py b/shelvery_tests/rds_integration_test.py index b48c5b0..8648c0c 100644 --- a/shelvery_tests/rds_integration_test.py +++ b/shelvery_tests/rds_integration_test.py @@ -1,17 +1,14 @@ import sys -import traceback -from turtle import back import unittest import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime -from shelvery import aws_helper -from shelvery_tests.test_functions import addBackupTags, compareBackups, initCleanup, initCreateBackups, initSetup, initShareBackups, instanceCleanupBackups, instanceShareBackups +from botocore.exceptions import WaiterError +from shelvery.engine import ShelveryEngine +from shelvery.runtime_config import RuntimeConfig +from shelvery_tests.test_functions import setup_source, compare_backups +from shelvery.rds_backup import ShelveryRDSBackup +from shelvery.aws_helper import AwsHelper +from shelvery_tests.resources import RDS_INSTANCE_RESOURCE_NAME, ResourceClass pwd = os.path.dirname(os.path.abspath(__file__)) @@ -21,115 +18,153 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") -from shelvery.rds_backup import ShelveryRDSBackup -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper -from shelvery_tests.conftest import destination_account - print(f"Python lib path:\n{sys.path}") - +class RDSInstanceTestClass(ResourceClass): + + def __init__(self): + self.resource_name = RDS_INSTANCE_RESOURCE_NAME + self.backups_engine = ShelveryRDSBackup() + self.client = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') + self.ARN = f"arn:aws:rds:{os.environ['AWS_DEFAULT_REGION']}:{AwsHelper.local_account_id()}:db:{self.resource_name}" + + def add_backup_tags(self): + self.client.add_tags_to_resource( + ResourceName=self.ARN, + Tags=[{ + 'Key': f"{RuntimeConfig.get_tag_prefix()}:{ShelveryEngine.BACKUP_RESOURCE_TAG}", + 'Value': 'true' + }, + {'Key': 'Name', + 'Value': self.resource_name + } + ] + ) + + def wait_for_resource(self): + waiter = AwsHelper.boto3_client('rds', region_name='ap-southeast-2').get_waiter('db_instance_available') + try: + waiter.wait( + DBInstanceIdentifier=self.resource_name, + WaiterConfig={ + 'Delay': 30, + 'MaxAttempts': 50 + } + ) + except WaiterError as error: + print("Waiting for RDS Instance Failed") + print(error) + raise error + +######## Test Case class ShelveryRDSIntegrationTestCase(unittest.TestCase): - """Shelvery RDS Backups Integration shelvery tests""" + """Shelvery RDS Instance Backups Integration shelvery tests""" def id(self): return str(self.__class__) - def setUp(self): + # Complete initial setup self.created_snapshots = [] - self.regional_snapshots = { - 'ap-southeast-1': [], - 'ap-southeast-2': [] - } - os.environ['SHELVERY_MONO_THREAD'] = '1' + setup_source(self) + # Instantiate resource test class + rds_instance_test_class = RDSInstanceTestClass() + # Wait till RDS Instance is in an available state + rds_instance_test_class.wait_for_resource() + # Add tags to indicate backup + rds_instance_test_class.add_backup_tags() - # Create and configure RDS artefact - initSetup(self,'rds') - rdsclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') + @pytest.mark.source + def test_CleanupRdsInstanceBackup(self): + print(f"RDS Instance - Running cleanup test") + # Create test resource class + rds_instance_test_class = RDSInstanceTestClass() + backups_engine = rds_instance_test_class.backups_engine + client = rds_instance_test_class.client + # Create backups + backups = backups_engine.create_backups() + # Clean backups + backups_engine.clean_backups() + # Retrieve remaining backups + snapshots = [ + snapshot + for backup in backups + for snapshot in client.describe_db_snapshots( + DBInstanceIdentifier=rds_instance_test_class.resource_name, + DBSnapshotIdentifier=backup.backup_id + )["DBSnapshots"] + ] + print(f"Snapshots: {snapshots}") - #Get db instance name - rdsinstance = rdsclient.describe_db_instances(DBInstanceIdentifier='shelvery-test-rds')['DBInstances'][0] + self.assertTrue(len(snapshots) == 0) - # add tags to resource - addBackupTags(rdsclient, - rdsinstance['DBInstanceArn'], - "shelvery-test-rds") - - self.share_with_id = destination_account - @pytest.mark.source - def test_Cleanup(self): - print(f"rds - Running cleanup test") - rds_backups_engine = ShelveryRDSBackup() - backups = initCleanup(rds_backups_engine) - rds_client = AwsHelper.boto3_client('rds') - - valid = False - for backup in backups: - valid = instanceCleanupBackups(self=self, - backup=backup, - backup_engine=rds_backups_engine, - service_client=rds_client - ) + def test_CreateRdsInstanceBackup(self): + print("Running RDS Instance create backup test") + # Instantiate test resource class + rds_instance_test_class = RDSInstanceTestClass() + backups_engine = rds_instance_test_class.backups_engine - self.assertTrue(valid) - - @pytest.mark.source - def test_CreateRdsBackup(self): - print(f"rds - Running backup test") - - rds_backup_engine = ShelveryRDSBackup() - print(f"rds - Shelvery backup initialised") - - backups = initCreateBackups(rds_backup_engine) - print("Created RDS backups") - - valid = False - - # validate there is + # Create backups + backups = backups_engine.create_backups() + print(f"Created {len(backups)} backups for RDS Instance") + + # Compare backups for backup in backups: - valid = compareBackups(self=self, - backup=backup, - backup_engine=rds_backup_engine - ) - self.assertTrue(valid) - + valid = compare_backups(self=self, backup=backup, backup_engine=backups_engine) + + # Clean backups + print(f"Cleaning up RDS Instance Backups") + backups_engine.clean_backups() + + # Validate backups + self.assertTrue(valid, f"Backup {backup} is not valid") + + self.assertEqual(len(backups), 1, f"Expected 1 backup, but found {len(backups)}") + @pytest.mark.source @pytest.mark.share - def test_shareRdsBackup(self): + def test_shareRdsInstanceBackup(self): - print(f"rds - Running share backup test") - rds_backups_engine = ShelveryRDSBackup() + print("Running RDS Instance share backup test") - backups = initShareBackups(rds_backups_engine, str(self.share_with_id)) - - print("Shared backups created") - - valid = False - # validate there is - for backup in backups: - valid = instanceShareBackups(self=self, - backup=backup - ) - - self.assertTrue(valid) - - def tearDown(self): - print("rds - tear down rds instance") - rdsclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') - for snapid in self.created_snapshots: - print(f"Deleting snapshot {snapid}") - try: - rdsclient.delete_db_snapshot(DBSnapshotIdentifier=snapid) - except Exception as e: - print(f"Failed to delete {snapid}:{str(e)}") - - print("rds - snapshot deleted, instance deleting") + # Instantiate test resource class + rds_instance_test_class = RDSInstanceTestClass() + backups_engine = rds_instance_test_class.backups_engine + client = rds_instance_test_class.client + print("Creating shared backups") + backups = backups_engine.create_backups() + print(f"{len(backups)} shared backups created") + for backup in backups: + snapshot_id = backup.backup_id + print(f"Checking if snapshot {snapshot_id} is shared with {self.share_with_id}") + + # Retrieve remaining backups + snapshots = [ + snapshot + for backup in backups + for snapshot in client.describe_db_snapshots( + DBInstanceIdentifier=rds_instance_test_class.resource_name, + DBSnapshotIdentifier=backup.backup_id + )["DBSnapshots"] + ] + + # Get attributes of snapshot + attributes = client.describe_db_snapshot_attributes( + DBSnapshotIdentifier=snapshot_id + )['DBSnapshotAttributesResult']['DBSnapshotAttributes'] + + # Check if snapshot is shared with destination account + shared_with_destination = any( + attr['AttributeName'] == 'restore' and self.share_with_id in attr['AttributeValues'] + for attr in attributes + ) + + # Assertions + self.assertEqual(len(snapshots), 1, f"Expected 1 snapshot, but found {len(snapshots)}") + self.assertTrue(shared_with_destination, f"Snapshot {snapshot_id} is not shared with {self.share_with_id}") + if __name__ == '__main__': unittest.main() diff --git a/shelvery_tests/rds_pull_test.py b/shelvery_tests/rds_pull_test.py index 9224bed..9063acb 100644 --- a/shelvery_tests/rds_pull_test.py +++ b/shelvery_tests/rds_pull_test.py @@ -1,17 +1,10 @@ import sys -import traceback import unittest import pytest -import yaml - -import boto3 import os -import time -import botocore -from datetime import datetime -from shelvery.rds_backup import ShelveryRDSBackup - -from shelvery_tests.cleanup_functions import cleanRdsSnapshots +from shelvery_tests.rds_integration_test import RDSInstanceTestClass +from shelvery_tests.test_functions import setup_destination +from shelvery_tests.resources import RDS_INSTANCE_RESOURCE_NAME pwd = os.path.dirname(os.path.abspath(__file__)) @@ -21,51 +14,42 @@ sys.path.append(f"{pwd}/lib") sys.path.append(f"{pwd}/../lib") - -from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX -from shelvery.runtime_config import RuntimeConfig -from shelvery.backup_resource import BackupResource -from shelvery.aws_helper import AwsHelper -from shelvery_tests.conftest import source_account - -#Need to add 'source acc' to env -#Call create_data_bucket -#How to cleanup source account after running in dest? - class ShelveryRDSPullTestCase(unittest.TestCase): @pytest.mark.destination def test_PullRdsBackup(self): - os.environ['SHELVERY_MONO_THREAD'] = '1' - cleanRdsSnapshots() - - source_aws_id = source_account - os.environ["shelvery_source_aws_account_ids"] = str(source_aws_id) - - print(f"rds - Running pull shared backups test") - - rds_client = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') - rds_backup_engine = ShelveryRDSBackup() - + + # Complete initial setup + print(f"RDS Instance - Running pull shared backups test") + setup_destination(self) + + # Create test resource class + rds_instance_test_class = RDSInstanceTestClass() + backups_engine = rds_instance_test_class.backups_engine + client = rds_instance_test_class.client + + # Clean residual existing snapshots + backups_engine.clean_backups() - print("Pulling shared backups") - rds_backup_engine.pull_shared_backups() + # Pull shared backups + backups_engine.pull_shared_backups() - #Get post-pull snapshot count - pulled_snapshot = rds_client.describe_db_snapshots( - DBInstanceIdentifier='shelvery-test-rds', + # Get post-pull snapshot count + pulled_snapshots = client.describe_db_snapshots( + DBInstanceIdentifier=RDS_INSTANCE_RESOURCE_NAME, SnapshotType='Manual' ) - print("PULLED:" + str(pulled_snapshot)) - self.assertTrue(len(pulled_snapshot['DBSnapshots']) == 1) - - - + # Verify that only one snapshot was pulled + self.assertEqual(len(pulled_snapshots["DBSnapshots"]), 1) + @pytest.mark.cleanup def test_cleanup(self): - cleanRdsSnapshots() + # Instantiate test resource class + rds_instance_test_class = RDSInstanceTestClass() + backups_engine = rds_instance_test_class.backups_engine + # Clean backups + backups_engine.clean_backups() diff --git a/shelvery_tests/resources.py b/shelvery_tests/resources.py new file mode 100644 index 0000000..630624a --- /dev/null +++ b/shelvery_tests/resources.py @@ -0,0 +1,21 @@ +from abc import abstractmethod + +DOCDB_RESOURCE_NAME='shelvery-test-docdb' +RDS_INSTANCE_RESOURCE_NAME='shelvery-test-rds' +RDS_CLUSTER_RESOURCE_NAME='shelvery-test-rds-cluster' +EC2_AMI_INSTANCE_RESOURCE_NAME='shelvery-test-ec2' +EBS_INSTANCE_RESOURCE_NAME='shelvery-test-ebs' +class ResourceClass(): + + def __init__(self): + self.resource_name = None + self.backups_engine = None + self.client = None + + @abstractmethod + def add_backup_tags(self): + pass + + @abstractmethod + def wait_for_resource(self): + pass \ No newline at end of file diff --git a/shelvery_tests/test_functions.py b/shelvery_tests/test_functions.py index 75f621f..47f47fa 100644 --- a/shelvery_tests/test_functions.py +++ b/shelvery_tests/test_functions.py @@ -1,95 +1,42 @@ -#Functions used to compare snapshots/clusters/instances -from datetime import datetime -from turtle import back -from warnings import filters -from xml.dom import pulldom -from shelvery.backup_resource import BackupResource -from shelvery.engine import ShelveryEngine -from shelvery.runtime_config import RuntimeConfig -from shelvery.aws_helper import AwsHelper -import traceback -import sys import os import boto3 import time import yaml -import pytest from shelvery_tests.conftest import destination_account, source_account -from shelvery_tests.cleanup_functions import cleanEC2Snapshots -# Compare backup cluster with aws cluster - -# Compare rds snap with backup snap etc +from shelvery.runtime_config import RuntimeConfig +from shelvery.aws_helper import AwsHelper -def addBackupTags(client,resource_name,tag_value): - response = client.add_tags_to_resource( - ResourceName=resource_name, - Tags=[{ - 'Key': f"{RuntimeConfig.get_tag_prefix()}:{ShelveryEngine.BACKUP_RESOURCE_TAG}", - 'Value': 'true' - }, {'Key': 'Name', 'Value': tag_value}] - ) +def setup_source(self): + print(f"Setting up integration test") + self.share_with_id = destination_account + os.environ["shelvery_share_aws_account_ids"] = destination_account + os.environ['AWS_DEFAULT_REGION'] = 'ap-southeast-2' + os.environ['SHELVERY_MONO_THREAD'] = '1' + os.environ['shelvery_custom_retention_types'] = 'shortLived:1' + os.environ['shelvery_current_retention_type'] = 'shortLived' - return response + sts = AwsHelper.boto3_client('sts') + self.id = sts.get_caller_identity() + print(f"Running as user:\n{self.id}\n") -def createBackupTags(client,resource_list,tag_value): - response = client.create_tags( - Resources=resource_list, - Tags=[{ - 'Key': f"{RuntimeConfig.get_tag_prefix()}:{ShelveryEngine.BACKUP_RESOURCE_TAG}", - 'Value': 'true' - }, {'Key': 'Name', 'Value': tag_value}] - ) - return response +def setup_destination(self): + print(f"Setting up integration test") -def initSetup(self,service_name): - print(f"Setting up {service_name} integration test") os.environ['AWS_DEFAULT_REGION'] = 'ap-southeast-2' os.environ['SHELVERY_MONO_THREAD'] = '1' + os.environ['shelvery_custom_retention_types'] = 'shortLived:1' + os.environ['shelvery_current_retention_type'] = 'shortLived' + os.environ["shelvery_source_aws_account_ids"] = source_account sts = AwsHelper.boto3_client('sts') self.id = sts.get_caller_identity() print(f"Running as user:\n{self.id}\n") -def initCreateBackups(backup_engine): - try: - backups = backup_engine.create_backups() - except Exception as e: - print(e) - print(f"Failed with {e}") - traceback.print_exc(file=sys.stdout) - raise e - - return backups - -def initShareBackups(backup_engine, share_with_id): - try: - os.environ["shelvery_share_aws_account_ids"] = str(share_with_id) - backups = backup_engine.create_backups() - except Exception as e: - print(e) - print(f"Failed with {e}") - traceback.print_exc(file=sys.stdout) - raise e - finally: - del os.environ["shelvery_share_aws_account_ids"] - - return backups - -def initCleanup(backup_engine): - try: - backups = backup_engine.create_backups() - except Exception as e: - print(e) - print(f"Failed with {e}") - traceback.print_exc(file=sys.stdout) - raise e - - return backups - -def compareBackups(self,backup,backup_engine): +def compare_backups(self,backup,backup_engine): print("Inside backup loop" + backup.backup_id) snapshot_id = backup.backup_id self.created_snapshots.append(snapshot_id) + print("Snapshot:" + str(snapshot_id)) # wait for snapshot to become available backup_engine.wait_backup_available(backup.region, backup.backup_id, None, None) @@ -103,7 +50,7 @@ def compareBackups(self,backup,backup_engine): account_id = backup_engine.account_id s3path = f"backups/{backup_engine.get_engine_type()}/{engine_backup.name}.yaml" s3bucket = backup_engine.get_local_bucket_name() - print(f"Usingbucket {s3bucket}") + print(f"Using bucket {s3bucket}") print(f"Using path {s3path}") bucket = boto3.resource('s3').Bucket(s3bucket) object = bucket.Object(s3path) @@ -121,348 +68,4 @@ def compareBackups(self,backup,backup_engine): engine_backup.tags[f"{RuntimeConfig.get_tag_prefix()}:{tag}"] ) - return True - -def clusterShareBackups(self, backup, service): - print(f"BackupId={backup.backup_id}") - print(f"Accountd={backup.account_id}") - - snapshot_id = backup.backup_id - print(f"Testing if snapshot {snapshot_id} is shared with {self.share_with_id}") - - source_client = AwsHelper.boto3_client(service) - - #Get source snapshot - source_snapshot = source_client.describe_db_cluster_snapshots( - DBClusterIdentifier=backup.entity_id, - DBClusterSnapshotIdentifier=snapshot_id - ) - - attributes = source_client.describe_db_cluster_snapshot_attributes( - DBClusterSnapshotIdentifier=snapshot_id - )['DBClusterSnapshotAttributesResult']['DBClusterSnapshotAttributes'] - - #Restore attribute indicating restoreable snapshot - restore_attribute = [attr for attr in attributes if attr['AttributeName'] == 'restore'][0]['AttributeValues'] - - print("Attributes: " + str(restore_attribute)) - - #Assert Snapshot(s) exist - self.assertTrue(len(source_snapshot['DBClusterSnapshots']) ==1) - - #Assert that snapshot is shared with dest account - self.assertTrue(destination_account in restore_attribute) - - return True - -def clusterCleanupBackups(self, backup, backup_engine, resource_client): - snapshot_id = backup.backup_id - - #Get source snapshot - precleanup_snapshots = resource_client.describe_db_cluster_snapshots( - DBClusterIdentifier=backup.entity_id, - DBClusterSnapshotIdentifier=snapshot_id - ) - - #Snapshot before cleanup - print("Pre-Cleanup Snapshots: " + str(precleanup_snapshots)) - - #Assert Snapshot(s) exist - self.assertTrue(len(precleanup_snapshots['DBClusterSnapshots']) ==1) - - snapshot_arn = precleanup_snapshots['DBClusterSnapshots'][0]['DBClusterSnapshotArn'] - - #Set cleanup date - resource_client.add_tags_to_resource( - ResourceName=snapshot_arn, - Tags=[{'Key': f"{RuntimeConfig.get_tag_prefix()}:date_created", - 'Value': datetime(1990, 1, 1).strftime(BackupResource.TIMESTAMP_FORMAT) - }] - ) - - backup_engine.clean_backups() - - #Get post cleanup snapshots - postcleanup_snapshots = resource_client.describe_db_cluster_snapshots( - DBClusterIdentifier=backup.entity_id, - DBClusterSnapshotIdentifier=snapshot_id - ) - - #Snapshot after cleanup - print("Post-Cleanup Snapshots: " + str(postcleanup_snapshots)) - - #Ensure cleanup removed all snapshots - self.assertTrue(len(postcleanup_snapshots['DBClusterSnapshots']) == 0) - - return True - -def instanceShareBackups(self,backup): - print(f"BackupId={backup.backup_id}") - print(f"Accountd={backup.account_id}") - - snapshot_id = backup.backup_id - print(f"Testing if snapshot {snapshot_id} is shared with {self.share_with_id}") - - source_client = AwsHelper.boto3_client('rds') - - #Get source snapshot - source_snapshot = source_client.describe_db_snapshots( - DBInstanceIdentifier=backup.entity_id, - DBSnapshotIdentifier=snapshot_id - ) - - attributes = source_client.describe_db_snapshot_attributes( - DBSnapshotIdentifier=snapshot_id - )['DBSnapshotAttributesResult']['DBSnapshotAttributes'] - - #Restore attribute indicating restoreable snapshot - restore_attribute = [attr for attr in attributes if attr['AttributeName'] == 'restore'][0]['AttributeValues'] - - print("Attributes: " + str(restore_attribute)) - - #Assert Snapshot(s) exist - self.assertTrue(len(source_snapshot['DBSnapshots']) ==1) - - #Assert that snapshot is shared with dest account - self.assertTrue(destination_account in restore_attribute) - - return True - -def instanceCleanupBackups(self,backup,backup_engine,service_client): - snapshot_id = backup.backup_id - - #Get source snapshot - precleanup_snapshots = service_client.describe_db_snapshots( - DBInstanceIdentifier=backup.entity_id, - DBSnapshotIdentifier=snapshot_id - ) - - #Snapshot before cleanup - print("Pre-Cleanup Snapshots: " + str(precleanup_snapshots)) - - #Assert Snapshot(s) exist - self.assertTrue(len(precleanup_snapshots['DBSnapshots']) ==1) - - snapshot_arn = precleanup_snapshots['DBSnapshots'][0]['DBSnapshotArn'] - - #Set cleanup date - service_client.add_tags_to_resource( - ResourceName=snapshot_arn, - Tags=[{'Key': f"{RuntimeConfig.get_tag_prefix()}:date_created", - 'Value': datetime(1990, 1, 1).strftime(BackupResource.TIMESTAMP_FORMAT) - }] - ) - - backup_engine.clean_backups() - - #Get post cleanup snapshots - postcleanup_snapshots = service_client.describe_db_snapshots( - DBInstanceIdentifier=backup.entity_id, - DBSnapshotIdentifier=snapshot_id - ) - - #Snapshot after cleanup - print("Post-Cleanup Snapshots: " + str(postcleanup_snapshots)) - - #Ensure cleanup removed all snapshots - self.assertTrue(len(postcleanup_snapshots['DBSnapshots']) == 0) - - return True - -def ec2ShareBackups(self,backup): - print(f"BackupId={backup.backup_id}") - print(f"Accountd={backup.account_id}") - - snapshot_name = backup.name - - source_client = AwsHelper.boto3_client('ec2') - - print(f"Testing if snapshot {snapshot_name} is shared with {self.share_with_id}") - - #Get source snapshot - source_snapshot = source_client.describe_snapshots( - Filters = [{ - 'Name': 'tag:Name', - 'Values': [ - snapshot_name - ] - }] - ) - - snapshot_id = source_snapshot['Snapshots'][0]['SnapshotId'] - - attributes = source_client.describe_snapshot_attribute( - Attribute='createVolumePermission', - SnapshotId=snapshot_id, - )['CreateVolumePermissions'] - - #Restore attribute indicating restoreable snapshot - restore_attribute = [attr for attr in attributes if attr['UserId'] == destination_account] - - print("Attributes: " + str(restore_attribute)) - - #Assert Snapshot(s) exist - self.assertTrue(len(source_snapshot['Snapshots']) ==1) - - #Assert that snapshot is shared with dest account - self.assertTrue(len(restore_attribute) == 1) - - return True - -def ebsCleanupBackups(self,backup,backup_engine,service_client): - snapshot_id = backup.name - - #Get pre-cleanup snapshots - precleanup_snapshots = service_client.describe_snapshots( - Filters = [{ - 'Name': 'tag:Name', - 'Values': [ - snapshot_id - ] - }] - ) - - #Assert Snapshot(s) exist - self.assertTrue(len(precleanup_snapshots['Snapshots']) ==1) - - #Create snapshot id - snap_id = precleanup_snapshots['Snapshots'][0]['SnapshotId'] - - #Set cleanup date - service_client.create_tags( - Resources=[snap_id], - Tags=[{'Key': f"{RuntimeConfig.get_tag_prefix()}:date_created", - 'Value': datetime(1990, 1, 1).strftime(BackupResource.TIMESTAMP_FORMAT) - }] - ) - - backup_engine.clean_backups() - - #Get Snapshots after cleanup - postcleanup_snapshots = service_client.describe_snapshots( - Filters = [{ - 'Name': 'tag:Name', - 'Values': [ - snapshot_id - ] - }] - ) - - #Snapshot after cleanup - print("Post-Cleanup Snapshots: " + str(postcleanup_snapshots)) - - #Ensure cleanup removed all snapshots - self.assertTrue(len(postcleanup_snapshots['Snapshots']) == 0) - - return True - -def ec2CleanupBackups(self,backup,backup_engine,service_client): - snapshot_id = backup.name - - #Get pre-cleanup snapshots - precleanup_snapshots = service_client.describe_snapshots( - Filters = [{ - 'Name': 'tag:Name', - 'Values': [ - snapshot_id - ] - }] - ) - - #Assert Snapshot(s) exist - self.assertTrue(len(precleanup_snapshots['Snapshots']) ==1) - - tags= precleanup_snapshots['Snapshots'][0]['Tags'] - - ami_id = [tag['Value'] for tag in tags if tag['Key'] == 'shelvery:ami_id'][0] - - #Create snapshot id - snap_id = precleanup_snapshots['Snapshots'][0]['SnapshotId'] - - #Set cleanup date - service_client.create_tags( - Resources=[snap_id,ami_id], - Tags=[{'Key': f"{RuntimeConfig.get_tag_prefix()}:date_created", - 'Value': datetime(1990, 1, 1).strftime(BackupResource.TIMESTAMP_FORMAT) - }] - ) - - backup_engine.clean_backups() - - #Get Snapshots after cleanup - postcleanup_snapshots = service_client.describe_snapshots( - Filters = [{ - 'Name': 'tag:Name', - 'Values': [ - snapshot_id - ] - }] - ) - - #Snapshot after cleanup - print("Post-Cleanup Snapshots: " + str(postcleanup_snapshots)) - - #Ensure cleanup removed all snapshots - self.assertTrue(len(postcleanup_snapshots['Snapshots']) == 0) - - return True - -def ebsPullBackups(self, service_client, backup_engine, db_identifier): - - cleanEC2Snapshots() - - #Set environment variables - source_aws_id = source_account - os.environ["shelvery_source_aws_account_ids"] = str(source_aws_id) - - print("Pulling shared backups") - backup_engine.pull_shared_backups() - - #Get owned snapshots - owned_snapshots = service_client.describe_snapshots( - OwnerIds=[ - destination_account, - ] - )['Snapshots'] - - print("Owned Snapshots: " + str(owned_snapshots)) - - pulled_snapshots = [] - - #Retrieve all snapshots with 'shelvery-test-ebs' in tags - for snapshot in owned_snapshots: - if 'Tags' in snapshot: - tags = snapshot['Tags'] - name = [tag['Value'] for tag in tags if tag['Key'] == 'Name'][0] - - if 'shelvery-test-ebs' in name: - pulled_snapshots.append(snapshot) - - #Assert 1 image has been pulled - print(pulled_snapshots) - self.assertTrue(len(pulled_snapshots) == 1) - -def ec2PullBackups(self, service_client, backup_engine): - - cleanEC2Snapshots() - - #Set environment variables - source_aws_id = source_account - os.environ["shelvery_source_aws_account_ids"] = str(source_aws_id) - - backup_engine.pull_shared_backups() - - search_filter = [{'Name':'tag:ResourceName', - 'Values':['shelvery-test-ec2'] - }] - - #Retrieve pulled images from shelvery-test stack - amis = service_client.describe_images( - Filters=search_filter - )['Images'] - - print("AMI's: " + str(amis)) - - #Ensure 1 image has been pulled - self.assertTrue(len(amis) == 1) - + return True \ No newline at end of file diff --git a/shelvery_tests/zname_transformation_test.py b/shelvery_tests/zname_transformation_test.py index ae437ed..555edb8 100644 --- a/shelvery_tests/zname_transformation_test.py +++ b/shelvery_tests/zname_transformation_test.py @@ -1,14 +1,9 @@ import sys import traceback import unittest -import yaml -import boto3 import os import time -import botocore import pytest -from datetime import datetime - pwd = os.path.dirname(os.path.abspath(__file__)) @@ -20,7 +15,6 @@ from shelvery.ebs_backup import ShelveryEBSBackup from shelvery.engine import ShelveryEngine -from shelvery.engine import S3_DATA_PREFIX from shelvery.runtime_config import RuntimeConfig from shelvery.backup_resource import BackupResource from shelvery.aws_helper import AwsHelper