From 523066f6561b13924d60bc82d9a76b6bba684282 Mon Sep 17 00:00:00 2001 From: Trent Smith Date: Mon, 4 Nov 2019 14:36:26 -0800 Subject: [PATCH 1/7] WIP backup code --- fusillade/clouddirectory.py | 9 +++++ scripts/backup.py | 74 +++++++++++++++++++++++++++++++++++++ tests/test_backup.py | 29 +++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 scripts/backup.py create mode 100644 tests/test_backup.py diff --git a/fusillade/clouddirectory.py b/fusillade/clouddirectory.py index 3e49a3e3..725ab377 100644 --- a/fusillade/clouddirectory.py +++ b/fusillade/clouddirectory.py @@ -1270,6 +1270,15 @@ def _exists(cls, nodes: List[str]): except cd_client.exceptions.ResourceNotFoundException: raise FusilladeBadRequestException(f"One or more {cls.object_type} does not exist.") + def list_owners(self, incoming=True): + get_links = self.cd.list_incoming_typed_links if incoming else self.cd.list_outgoing_typed_links + object_selection = 'SourceObjectReference' if incoming else 'TargetObjectReference' + return [ + type_link[object_selection]['Selector'] + for type_link in + get_links(self.object_ref, filter_typed_link='ownership_link') + ] + class PolicyMixin: """Adds policy support to a cloudNode""" diff --git a/scripts/backup.py b/scripts/backup.py new file mode 100644 index 00000000..1582bbd0 --- /dev/null +++ b/scripts/backup.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +""" +Create a backup of Clouddirectory in AWS S3. +""" +import os +import sys + +pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # noqa +sys.path.insert(0, pkg_root) # noqa + +from fusillade.clouddirectory import User, Group, Role + + +def list_node(node, field): + result, next_token = node.list_all(None, None) + while True: + for i in result[field]: + yield i + if next_token: + result, next_token = node.list_all(next_token, None) + else: + break + + +def backup_users(): + users = [] + # TODO list all users + for name in list_node(User, 'users'): + user = User(name) + info = { + 'name': user.name, + 'status': user.status, + 'policies': user.attached_policies} + users.append(info) + print(info) + return users + + +def backup_groups(): + groups = [] + for name in list_node(Group, 'groups'): + group = Group(name) + info = { + 'name': group.name, + 'members': [User(object_ref=u).name for u in group.get_users_iter()], + 'policies': group.attached_policies, + 'owners': [User(object_ref=u).name for u in group.list_owners()] + } + groups.append(info) + print(info) + return groups + +def backup_roles(): + roles = [] + for name in list_node(Role, 'roles'): + role = Role(name) + members = role.cd.list_object_children(role.object_ref) + for member in members: + + info = { + 'name': role.name, + 'owners': role.list_owners(), + } + +def backup(): + # users = backup_users() + # groups = backup_groups() + + # TODO backup roles + +# TODO we need backup format +# TODO we need a location to backup too + +backup() \ No newline at end of file diff --git a/tests/test_backup.py b/tests/test_backup.py new file mode 100644 index 00000000..57863bf4 --- /dev/null +++ b/tests/test_backup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# coding: utf-8 + +""" +Functional Test of the API +""" +import json +import os +import sys +import unittest + +from furl import furl + +pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # noqa +sys.path.insert(0, pkg_root) # noqa + +from tests import eventually +from tests.base_api_test import BaseAPITest +from tests.common import get_auth_header, service_accounts + + +class TestApi(BaseAPITest, unittest.TestCase): + + # TODO we need a location to backup too + # TODO we need backup format + + +if __name__ == '__main__': + unittest.main() From 5b1c6e49a6ea8ac7dc9fbf45fc301c5c8b7134ef Mon Sep 17 00:00:00 2001 From: Trent Smith Date: Mon, 4 Nov 2019 16:35:45 -0800 Subject: [PATCH 2/7] Backup the current clouddirectory to a file --- fusillade/clouddirectory.py | 15 ++++++++++--- scripts/backup.py | 42 ++++++++++++++++++++++--------------- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/fusillade/clouddirectory.py b/fusillade/clouddirectory.py index 725ab377..7ff72e17 100644 --- a/fusillade/clouddirectory.py +++ b/fusillade/clouddirectory.py @@ -354,6 +354,15 @@ def list_policy_attachments(self, policy: str, **kwargs) -> Iterator[str]: **kwargs ) + def list_object_parent_paths(self, object_ref: str, **kwargs) -> Iterator[str]: + return _paging_loop(cd_client.list_object_parent_paths, + 'PathToObjectIdentifiersList', + DirectoryArn=self._dir_arn, + ObjectReference={'Selector': object_ref}, + MaxResults=self._page_limit, + **kwargs + ) + @retry(**cd_read_retry_parameters) def _list_typed_links(self, func: Callable, @@ -423,7 +432,7 @@ def list_incoming_typed_links( @staticmethod def _make_ref(i): - return '$' + i + return i if i[0] == '$' else '$' + i def create_object(self, link_name: str, facet_type: str, obj_type: str, **kwargs) -> str: """ @@ -1040,7 +1049,7 @@ class CloudNode: Contains shared code across the different types of nodes stored in Fusillade CloudDirectory """ _attributes = ["name"] # the different attributes of a node stored - _facet = 'LeafNode' + _facet = 'LeafFacet' object_type = 'node' def __init__(self, @@ -1365,7 +1374,7 @@ def get_policy(self, policy_type: str = 'IAMPolicy'): self.attached_policies[policy_type] = attrs['policy_document'].decode("utf-8") except cd_client.exceptions.ResourceNotFoundException: pass - return self.attached_policies.get(policy_type, '') + return self.attached_policies.get(policy_type, '{}') else: FusilladeHTTPException( title='Bad Request', diff --git a/scripts/backup.py b/scripts/backup.py index 1582bbd0..1b46ac7d 100644 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -2,6 +2,7 @@ """ Create a backup of Clouddirectory in AWS S3. """ +import json import os import sys @@ -24,15 +25,16 @@ def list_node(node, field): def backup_users(): users = [] - # TODO list all users for name in list_node(User, 'users'): user = User(name) info = { 'name': user.name, 'status': user.status, - 'policies': user.attached_policies} + 'policies': [{p: json.loads(user.get_policy(p))} for p in user.allowed_policy_types], + 'roles': [Role(object_ref=r).name for r in user.roles] + } users.append(info) - print(info) + print("USERS:", *users, sep='\n\t') return users @@ -43,32 +45,38 @@ def backup_groups(): info = { 'name': group.name, 'members': [User(object_ref=u).name for u in group.get_users_iter()], - 'policies': group.attached_policies, - 'owners': [User(object_ref=u).name for u in group.list_owners()] + 'policies': [{p: json.loads(group.get_policy(p))} for p in group.allowed_policy_types], + 'owners': [User(object_ref=u).name for u in group.list_owners()], + 'roles': [Role(object_ref=r).name for r in group.roles] } groups.append(info) - print(info) + print("GROUPS:", *groups, sep='\n\t') return groups + def backup_roles(): roles = [] for name in list_node(Role, 'roles'): role = Role(name) - members = role.cd.list_object_children(role.object_ref) - for member in members: - info = { 'name': role.name, - 'owners': role.list_owners(), + 'owners': [User(object_ref=u).name for u in role.list_owners()], + 'policies': [{p: json.loads(role.get_policy(p))} for p in role.allowed_policy_types] } + roles.append(info) + print("ROLES:", *roles, sep='\n\t') + return roles -def backup(): - # users = backup_users() - # groups = backup_groups() - # TODO backup roles +def backup(): + with open('backup.json', 'w') as fp: + json.dump( + dict( + users=backup_users(), + groups=backup_groups(), + roles=backup_roles()), + fp, + indent=2) -# TODO we need backup format -# TODO we need a location to backup too -backup() \ No newline at end of file +backup() From 729f43be8ea23b677720099c312eb6a939513dc3 Mon Sep 17 00:00:00 2001 From: Trent Smith Date: Tue, 5 Nov 2019 09:28:42 -0800 Subject: [PATCH 3/7] Adding a way to determine the type of node based on parents This is used to determine the owner types. --- fusillade/clouddirectory.py | 3 ++- scripts/backup.py | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/fusillade/clouddirectory.py b/fusillade/clouddirectory.py index 7ff72e17..95980454 100644 --- a/fusillade/clouddirectory.py +++ b/fusillade/clouddirectory.py @@ -1266,6 +1266,7 @@ def list_all(cls, next_token: str, per_page: int): def get_info(self) -> Dict[str, Any]: info = dict(**self.get_attributes(self._attributes)) info[f'{self.object_type}_id'] = info.pop('name') + info['paths'] = [i['Path'] for i in self.cd.list_object_parent_paths(self.object_ref)] return info @classmethod @@ -1374,7 +1375,7 @@ def get_policy(self, policy_type: str = 'IAMPolicy'): self.attached_policies[policy_type] = attrs['policy_document'].decode("utf-8") except cd_client.exceptions.ResourceNotFoundException: pass - return self.attached_policies.get(policy_type, '{}') + return self.attached_policies.get(policy_type, '') else: FusilladeHTTPException( title='Bad Request', diff --git a/scripts/backup.py b/scripts/backup.py index 1b46ac7d..41ed8cf7 100644 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -1,15 +1,38 @@ #!/usr/bin/env python """ -Create a backup of Clouddirectory in AWS S3. +Create a backup of Clouddirectory and output the file ro backup.json """ + import json import os import sys +import typing pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # noqa sys.path.insert(0, pkg_root) # noqa -from fusillade.clouddirectory import User, Group, Role +from fusillade.clouddirectory import User, Group, Role, CloudNode + + +def format_policies(policies: typing.List[typing.Tuple[str, str]]) -> typing.Dict[str, str]: + rv = dict() + for t, p in policies: + if not p: + continue + else: + rv[t] = json.loads(p) + return rv + + +def format_owners(owned_node: CloudNode) -> typing.List[typing.Dict[str, str]]: + owners = [] + for u in owned_node.list_owners(): + node = CloudNode(object_ref=u) + owners.append({ + 'type': [i for i in [p.split('/')[1] for p in node.get_info()['paths']] if i != 'role'][0], + 'name': node.name + }) + return owners def list_node(node, field): @@ -30,7 +53,7 @@ def backup_users(): info = { 'name': user.name, 'status': user.status, - 'policies': [{p: json.loads(user.get_policy(p))} for p in user.allowed_policy_types], + 'policies': format_policies([(p, user.get_policy(p)) for p in user.allowed_policy_types]), 'roles': [Role(object_ref=r).name for r in user.roles] } users.append(info) @@ -45,8 +68,8 @@ def backup_groups(): info = { 'name': group.name, 'members': [User(object_ref=u).name for u in group.get_users_iter()], - 'policies': [{p: json.loads(group.get_policy(p))} for p in group.allowed_policy_types], - 'owners': [User(object_ref=u).name for u in group.list_owners()], + 'policies': format_policies([(p, group.get_policy(p)) for p in group.allowed_policy_types]), + 'owners': format_owners(group), 'roles': [Role(object_ref=r).name for r in group.roles] } groups.append(info) @@ -60,8 +83,8 @@ def backup_roles(): role = Role(name) info = { 'name': role.name, - 'owners': [User(object_ref=u).name for u in role.list_owners()], - 'policies': [{p: json.loads(role.get_policy(p))} for p in role.allowed_policy_types] + 'policies': format_policies([(p, role.get_policy(p)) for p in role.allowed_policy_types]), + 'owners': format_owners(role) } roles.append(info) print("ROLES:", *roles, sep='\n\t') From 88ff954f3b7e5e1d69deac5bf07dbd08303bec1b Mon Sep 17 00:00:00 2001 From: Trent Smith Date: Tue, 5 Nov 2019 09:32:09 -0800 Subject: [PATCH 4/7] cleanup --- tests/test_backup.py | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 tests/test_backup.py diff --git a/tests/test_backup.py b/tests/test_backup.py deleted file mode 100644 index 57863bf4..00000000 --- a/tests/test_backup.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -""" -Functional Test of the API -""" -import json -import os -import sys -import unittest - -from furl import furl - -pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # noqa -sys.path.insert(0, pkg_root) # noqa - -from tests import eventually -from tests.base_api_test import BaseAPITest -from tests.common import get_auth_header, service_accounts - - -class TestApi(BaseAPITest, unittest.TestCase): - - # TODO we need a location to backup too - # TODO we need backup format - - -if __name__ == '__main__': - unittest.main() From 3b77608b85b1fa5b9256f986f9d6f031842ed483 Mon Sep 17 00:00:00 2001 From: Trent Smith Date: Tue, 5 Nov 2019 11:19:30 -0800 Subject: [PATCH 5/7] make the world simple --- scripts/backup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/backup.py b/scripts/backup.py index 41ed8cf7..00ec9de1 100644 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -29,7 +29,9 @@ def format_owners(owned_node: CloudNode) -> typing.List[typing.Dict[str, str]]: for u in owned_node.list_owners(): node = CloudNode(object_ref=u) owners.append({ - 'type': [i for i in [p.split('/')[1] for p in node.get_info()['paths']] if i != 'role'][0], + 'type': [i for i in + [p['Path'].split('/')[1] + for p in node.cd.list_object_parent_paths(node.object_ref)] if i != 'role'][0], 'name': node.name }) return owners From f55b15c864236433e80ebe2b65d5284255799cdb Mon Sep 17 00:00:00 2001 From: Trent Smith Date: Tue, 5 Nov 2019 11:32:27 -0800 Subject: [PATCH 6/7] simplify code --- fusillade/clouddirectory.py | 13 +++++++++++-- scripts/backup.py | 17 ++--------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/fusillade/clouddirectory.py b/fusillade/clouddirectory.py index 95980454..da897250 100644 --- a/fusillade/clouddirectory.py +++ b/fusillade/clouddirectory.py @@ -1266,7 +1266,6 @@ def list_all(cls, next_token: str, per_page: int): def get_info(self) -> Dict[str, Any]: info = dict(**self.get_attributes(self._attributes)) info[f'{self.object_type}_id'] = info.pop('name') - info['paths'] = [i['Path'] for i in self.cd.list_object_parent_paths(self.object_ref)] return info @classmethod @@ -1283,11 +1282,21 @@ def _exists(cls, nodes: List[str]): def list_owners(self, incoming=True): get_links = self.cd.list_incoming_typed_links if incoming else self.cd.list_outgoing_typed_links object_selection = 'SourceObjectReference' if incoming else 'TargetObjectReference' - return [ + _owners = [ type_link[object_selection]['Selector'] for type_link in get_links(self.object_ref, filter_typed_link='ownership_link') ] + owners = [] + for owner in _owners: + node = CloudNode(object_ref=owner) + owners.append({ + 'type': [i for i in + [p['Path'].split('/')[1] + for p in node.cd.list_object_parent_paths(node.object_ref)] if i != 'role'][0], + 'name': node.name + }) + return owners class PolicyMixin: diff --git a/scripts/backup.py b/scripts/backup.py index 00ec9de1..04e71994 100644 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -24,19 +24,6 @@ def format_policies(policies: typing.List[typing.Tuple[str, str]]) -> typing.Dic return rv -def format_owners(owned_node: CloudNode) -> typing.List[typing.Dict[str, str]]: - owners = [] - for u in owned_node.list_owners(): - node = CloudNode(object_ref=u) - owners.append({ - 'type': [i for i in - [p['Path'].split('/')[1] - for p in node.cd.list_object_parent_paths(node.object_ref)] if i != 'role'][0], - 'name': node.name - }) - return owners - - def list_node(node, field): result, next_token = node.list_all(None, None) while True: @@ -71,7 +58,7 @@ def backup_groups(): 'name': group.name, 'members': [User(object_ref=u).name for u in group.get_users_iter()], 'policies': format_policies([(p, group.get_policy(p)) for p in group.allowed_policy_types]), - 'owners': format_owners(group), + 'owners': group.list_owners(), 'roles': [Role(object_ref=r).name for r in group.roles] } groups.append(info) @@ -86,7 +73,7 @@ def backup_roles(): info = { 'name': role.name, 'policies': format_policies([(p, role.get_policy(p)) for p in role.allowed_policy_types]), - 'owners': format_owners(role) + 'owners': role.list_owners() } roles.append(info) print("ROLES:", *roles, sep='\n\t') From dac455f1ca34572693daf6d0bb2075a7ceac0741 Mon Sep 17 00:00:00 2001 From: Trent Smith <1429913+Bento007@users.noreply.github.com> Date: Tue, 5 Nov 2019 11:33:20 -0800 Subject: [PATCH 7/7] Apply suggestions from code review Co-Authored-By: amar jandu --- scripts/backup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/backup.py b/scripts/backup.py index 04e71994..8404e949 100644 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -91,4 +91,5 @@ def backup(): indent=2) -backup() +add if __name__ == "__main__": + backup()