-
Notifications
You must be signed in to change notification settings - Fork 22
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
* exlude docdb from rds cluster backups * Feature/unit tests refactor (#146) * Initial add * Disabled non docdb tests * Disabled non doc db stack resources * Fixed up set up test func * Changed waiter to rds * Fixed client call * Enable docdb pull test * Fixed bugs * Fixed typo * Fixed share id * Fixed docdb pull test * Disable docdb, enable rds instance * Fixed rds waiter * Enabled cloudformation for RDS Instance * Updated refactored unit tests * Removed test teardowns * Moved cleanup before assertion * Added fix for rds cluster docdb cleanup * Refactored unit tests * Re-enabled stack create * Removed bucket delete * Moved s3Upload outside script block in pipeline * Moved s3upload back into script block
- Loading branch information
1 parent
140b498
commit e84f8f4
Showing
18 changed files
with
913 additions
and
1,262 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,117 +1,53 @@ | ||
| from shelvery.documentdb_backup import ShelveryDocumentDbBackup | ||
| from shelvery.ebs_backup import ShelveryEBSBackup | ||
| from shelvery.ec2ami_backup import ShelveryEC2AMIBackup | ||
| from shelvery.rds_cluster_backup import ShelveryRDSClusterBackup | ||
| from shelvery.rds_backup import ShelveryRDSBackup | ||
| from shelvery.aws_helper import AwsHelper | ||
| import boto3 | ||
| import os | ||
|
|
||
| def cleanDocDBSnapshots(): | ||
| print("Cleaning up DocDB Snapshots") | ||
|
|
||
| docdbclient = AwsHelper.boto3_client('docdb', region_name='ap-southeast-2') | ||
|
|
||
| snapshots = docdbclient.describe_db_cluster_snapshots( | ||
| DBClusterIdentifier='shelvery-test-docdb', | ||
| SnapshotType='Manual' | ||
| )['DBClusterSnapshots'] | ||
|
|
||
| for snapshot in snapshots: | ||
| snapid = snapshot['DBClusterSnapshotIdentifier'] | ||
|
|
||
| try: | ||
| print(f"Deleting snapshot: {snapid}") | ||
| docdbclient.delete_db_cluster_snapshot(DBClusterSnapshotIdentifier=snapid) | ||
| except Exception as e: | ||
| print(f"Failed to delete {snapid}:{str(e)}") | ||
|
|
||
| backups_engine = ShelveryDocumentDbBackup() | ||
| backups_engine.clean_backups() | ||
|
|
||
| def cleanRdsClusterSnapshots(): | ||
| print("Cleaning up RDS Cluster Snapshots") | ||
| rdsclusterclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') | ||
|
|
||
| snapshots = rdsclusterclient.describe_db_cluster_snapshots( | ||
| DBClusterIdentifier='shelvery-test-rds-cluster', | ||
| SnapshotType='Manual' | ||
| )['DBClusterSnapshots'] | ||
|
|
||
| for snapshot in snapshots: | ||
| snapid = snapshot['DBClusterSnapshotIdentifier'] | ||
|
|
||
| try: | ||
| print(f"Deleting snapshot: {snapid}") | ||
| rdsclusterclient.delete_db_cluster_snapshot(DBClusterSnapshotIdentifier=snapid) | ||
| except Exception as e: | ||
| print(f"Failed to delete {snapid}:{str(e)}") | ||
| backups_engine = ShelveryRDSClusterBackup() | ||
| backups_engine.clean_backups() | ||
|
|
||
| def cleanRdsSnapshots(): | ||
| print("Cleaning up RDS Snapshots") | ||
| rdsclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2') | ||
|
|
||
| snapshots = rdsclient.describe_db_snapshots( | ||
| DBInstanceIdentifier='shelvery-test-rds', | ||
| SnapshotType='Manual', | ||
| )['DBSnapshots'] | ||
|
|
||
| for snapshot in snapshots: | ||
| snapid = snapshot['DBSnapshotIdentifier'] | ||
|
|
||
| try: | ||
| print(f"Deleting snapshot: {snapid}") | ||
| rdsclient.delete_db_snapshot(DBSnapshotIdentifier=snapid) | ||
| except Exception as e: | ||
| print(f"Failed to delete {snapid}:{str(e)}") | ||
| backups_engine = ShelveryRDSBackup() | ||
| backups_engine.clean_backups() | ||
|
|
||
| def cleanEC2Snapshots(): | ||
| #EC2 AMI | ||
| ec2client = AwsHelper.boto3_client('ec2', region_name='ap-southeast-2') | ||
| sts = AwsHelper.boto3_client('sts') | ||
| id = str(sts.get_caller_identity()['Account']) | ||
|
|
||
| snapshots = ec2client.describe_snapshots( | ||
| OwnerIds=[id] | ||
| )['Snapshots'] | ||
|
|
||
| for snapshot in snapshots: | ||
| snapid = snapshot['SnapshotId'] | ||
| if 'Tags' in snapshot: | ||
| tags = snapshot['Tags'] | ||
| try: | ||
| name = [tag['Value'] for tag in tags if tag['Key'] == 'Name'][0] | ||
| if 'shelvery-test-ec2' in name: | ||
| print("Cleaning up EC2 AMI Snapshots") | ||
| ami_id = [tag['Value'] for tag in tags if tag['Key'] == 'shelvery:ami_id'][0] | ||
| if ami_id != []: | ||
| print(f"De-registering image: {ami_id}") | ||
| ec2client.deregister_image(ImageId=ami_id) | ||
| ec2client.delete_snapshot(SnapshotId=snapid) | ||
| print(f'Deleting EC2 snapshot: {snapid}') | ||
| if 'shelvery-test-ebs' in name: | ||
| print("Cleaning up EBS Snapshots") | ||
| print(f'Deleting EBS snapshot: {snapid}') | ||
| ec2client.delete_snapshot(SnapshotId=snapid) | ||
| except Exception as e: | ||
| print(f"Failed to delete {snapid}:{str(e)}") | ||
|
|
||
| else: | ||
| print(f'Deleting Untagged EC2 Snapshots') | ||
| if snapshot['VolumeId'] == 'vol-ffffffff' and 'Copied for' in snapshot['Description']: | ||
|
|
||
| search_filter = [{'Name':'block-device-mapping.snapshot-id', | ||
| 'Values': [snapid], | ||
| 'Name':'tag:ResourceName', | ||
| 'Values':['shelvery-test-ec2'] | ||
| }] | ||
|
|
||
|
|
||
|
|
||
| ami_id = ec2client.describe_images( | ||
| Filters=search_filter | ||
| )['Images'][0]['ImageId'] | ||
| try: | ||
| print(f"De-registering image: {ami_id}") | ||
| print(f'Deleting EC2 snapshot: {snapid}') | ||
| ec2client.deregister_image(ImageId=ami_id) | ||
| ec2client.delete_snapshot(SnapshotId=snapid) | ||
| except Exception as e: | ||
| print(f"Failed to delete {snapid}:{str(e)}") | ||
| print("Cleaning up EC2 AMI Snapshots") | ||
| backups_engine = ShelveryEC2AMIBackup() | ||
| backups_engine.clean_backups() | ||
|
|
||
| def cleanEBSSnapshots(): | ||
| print("Cleaning up EBS Snapshots") | ||
| backups_engine = ShelveryEBSBackup() | ||
| backups_engine.clean_backups() | ||
|
|
||
| def cleanS3Bucket(): | ||
| print("Cleaning S3 Bucket") | ||
| bucket_name = f"shelvery.data.{AwsHelper.local_account_id()}-ap-southeast-2.base2tools" | ||
| s3 = boto3.resource('s3') | ||
| bucket = s3.Bucket(bucket_name) | ||
|
|
||
| # Delete all objects in the bucket | ||
| for obj in bucket.objects.all(): | ||
| obj.delete() | ||
|
|
||
| def cleanupSnapshots(): | ||
| os.environ['shelvery_custom_retention_types'] = 'shortLived:1' | ||
| os.environ['shelvery_current_retention_type'] = 'shortLived' | ||
| cleanDocDBSnapshots() | ||
| cleanEC2Snapshots() | ||
| cleanEBSSnapshots() | ||
| cleanRdsClusterSnapshots() | ||
| cleanRdsSnapshots() | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.