From 5ea93b9e8f89a531d10083044f4eaf1cc81caeb3 Mon Sep 17 00:00:00 2001 From: kangwork Date: Thu, 8 Aug 2024 15:32:17 +0900 Subject: [PATCH] etc: apply the Black formatter Signed-off-by: kangwork --- .../connector/cloud_identity_connector.py | 3 +- src/plugin/connector/iam_connector.py | 4 +- src/plugin/connector/logging_connector.py | 30 +++++++++++---- .../resource_manager_v3_connector.py | 14 +++++-- src/plugin/main.py | 3 +- src/plugin/manager/__init__.py | 1 - src/plugin/manager/base.py | 38 +++++++++++++------ src/plugin/manager/iam/group_manager.py | 10 ++++- src/plugin/manager/iam/permission_manager.py | 26 +++++++++---- src/plugin/manager/iam/role_manager.py | 18 ++++++--- .../manager/iam/service_account_manager.py | 32 ++++++++++++---- src/plugin/utils/__init__.py | 1 - src/plugin/utils/error_handlers.py | 13 +++++-- src/setup.py | 7 +--- 14 files changed, 140 insertions(+), 60 deletions(-) diff --git a/src/plugin/connector/cloud_identity_connector.py b/src/plugin/connector/cloud_identity_connector.py index 0532828..c609180 100644 --- a/src/plugin/connector/cloud_identity_connector.py +++ b/src/plugin/connector/cloud_identity_connector.py @@ -21,7 +21,7 @@ def list_groups(self, customer_id): response = request.execute() groups.extend(response.get("groups", [])) request = self.client.groups().list_next( - previous_request=request, previous_response=response + previous_request=request, previous_response=response ) if request is None: break @@ -57,4 +57,3 @@ def list_memberships(self, parent): def get_membership(self, name): result = self.client.groups().memberships().get(name=name).execute() return result - diff --git a/src/plugin/connector/iam_connector.py b/src/plugin/connector/iam_connector.py index 43a90ca..057dd2d 100644 --- a/src/plugin/connector/iam_connector.py +++ b/src/plugin/connector/iam_connector.py @@ -39,7 +39,9 @@ def list_service_accounts(self, project_id: str = None) -> list: return service_accounts @api_retry_handler(default_response=[]) - def list_service_account_keys(self, service_account_email: str, project_id: str = None): + def list_service_account_keys( + self, service_account_email: str, project_id: str = None + ): project_id = project_id or self.project_id query = { "name": f"projects/{project_id}/serviceAccounts/{service_account_email}" diff --git a/src/plugin/connector/logging_connector.py b/src/plugin/connector/logging_connector.py index ea4e84c..02ae766 100644 --- a/src/plugin/connector/logging_connector.py +++ b/src/plugin/connector/logging_connector.py @@ -14,7 +14,9 @@ class LoggingConnector(GoogleCloudConnector): version = "v2" def __init__(self, options: dict, secret_data: dict, schema: str, *args, **kwargs): - super().__init__(options=options, secret_data=secret_data, schema=schema, *args, **kwargs) + super().__init__( + options=options, secret_data=secret_data, schema=schema, *args, **kwargs + ) self.log_search_period = options.get("log_search_period", "3 Months") @api_retry_handler(default_response=[]) @@ -29,21 +31,30 @@ def list_entries(self, project_id: str) -> list: entries = response.get("entries", []) return entries - def get_last_log_entry_timestamp(self, project_id: str, service_account_email: str, service_account_key_name: str=None): - log_entries = (self._list_entries_service_accounts(project_id, service_account_email, service_account_key_name)) + def get_last_log_entry_timestamp( + self, + project_id: str, + service_account_email: str, + service_account_key_name: str = None, + ): + log_entries = self._list_entries_service_accounts( + project_id, service_account_email, service_account_key_name + ) if not log_entries: return None timestamp = log_entries[0].get("timestamp") return timestamp @api_retry_handler(default_response=[]) - def _list_entries_service_accounts(self, project_id: str, service_account_email: str, service_account_key_name) -> list: + def _list_entries_service_accounts( + self, project_id: str, service_account_email: str, service_account_key_name + ) -> list: filter_str = ( - f"protoPayload.authenticationInfo.principalEmail=\"{service_account_email}\"" + f'protoPayload.authenticationInfo.principalEmail="{service_account_email}"' ) if service_account_key_name: - full_name = f"//iam.googleapis.com/{service_account_key_name}" - filter_str += f" AND protoPayload.authenticationInfo.serviceAccountKeyName=\"{full_name}\"" + resource_name = f"//iam.googleapis.com/{service_account_key_name}" + filter_str += f' AND protoPayload.authenticationInfo.serviceAccountKeyName="{resource_name}"' filter_str += self._get_timestamp_filter_str(datetime.utcnow()) body = { @@ -66,7 +77,10 @@ def _get_timestamp_filter_str(self, time_now: datetime) -> str: else: log_search_period_in_days = 365 - start_time, end_time = time_now - timedelta(days=log_search_period_in_days), time_now + start_time, end_time = ( + time_now - timedelta(days=log_search_period_in_days), + time_now, + ) start_time_str, end_time_str = ( start_time.isoformat().split(".")[0] + "Z", end_time.isoformat().split(".")[0] + "Z", diff --git a/src/plugin/connector/resource_manager_v3_connector.py b/src/plugin/connector/resource_manager_v3_connector.py index 92b6283..5cecc68 100644 --- a/src/plugin/connector/resource_manager_v3_connector.py +++ b/src/plugin/connector/resource_manager_v3_connector.py @@ -55,15 +55,23 @@ def get_project_iam_policies(self, project_id: str = None): project_id = project_id or self.project_id resource = f"projects/{project_id}" body = {"options": {"requestedPolicyVersion": 3}} - result = self.client.projects().getIamPolicy(resource=resource, body=body).execute() + result = ( + self.client.projects().getIamPolicy(resource=resource, body=body).execute() + ) return result.get("bindings", []) def get_folder_iam_policies(self, resource): body = {"options": {"requestedPolicyVersion": 3}} - result = self.client.folders().getIamPolicy(resource=resource, body=body).execute() + result = ( + self.client.folders().getIamPolicy(resource=resource, body=body).execute() + ) return result.get("bindings", []) def get_organization_iam_policies(self, resource): body = {"options": {"requestedPolicyVersion": 3}} - result = self.client.organizations().getIamPolicy(resource=resource, body=body).execute() + result = ( + self.client.organizations() + .getIamPolicy(resource=resource, body=body) + .execute() + ) return result.get("bindings", []) diff --git a/src/plugin/main.py b/src/plugin/main.py index 3e1b02b..355732d 100644 --- a/src/plugin/main.py +++ b/src/plugin/main.py @@ -48,7 +48,6 @@ def _create_init_metadata() -> dict: "inventory.Region", "inventory.ErrorResource", ], - "options_schema": { "required": ["log_search_period"], "type": "object", @@ -62,7 +61,7 @@ def _create_init_metadata() -> dict: "description": "Select the period to search for the last activity log for the service accounts.\ The longer the period, the longer it will take to collect data.", } - } + }, }, } } diff --git a/src/plugin/manager/__init__.py b/src/plugin/manager/__init__.py index 944c48e..7286670 100644 --- a/src/plugin/manager/__init__.py +++ b/src/plugin/manager/__init__.py @@ -1,2 +1 @@ from .iam import * - diff --git a/src/plugin/manager/base.py b/src/plugin/manager/base.py index 628fdd9..4c6c406 100644 --- a/src/plugin/manager/base.py +++ b/src/plugin/manager/base.py @@ -36,19 +36,29 @@ def __init__(self, *args, **kwargs): def __repr__(self): return f"{self.__class__.__name__}" - def collect_resources(self, options: dict, secret_data: dict, schema: str) -> Generator[dict, None, None]: + def collect_resources( + self, options: dict, secret_data: dict, schema: str + ) -> Generator[dict, None, None]: try: - _LOGGER.debug(f"[{self.__repr__()}] Collect cloud service type: " - f"{self.cloud_service_group} > {self.cloud_service_type}") + _LOGGER.debug( + f"[{self.__repr__()}] Collect cloud service type: " + f"{self.cloud_service_group} > {self.cloud_service_type}" + ) yield self.get_cloud_service_type() - _LOGGER.debug(f"[{self.__repr__()}] Collect metrics: " - f"{self.cloud_service_group} > {self.cloud_service_type}") + _LOGGER.debug( + f"[{self.__repr__()}] Collect metrics: " + f"{self.cloud_service_group} > {self.cloud_service_type}" + ) yield from self.collect_metrics() - _LOGGER.debug(f"[{self.__repr__()}] Collect cloud services: " - f"{self.cloud_service_group} > {self.cloud_service_type}") - response_iterator = self.collect_cloud_services(options, secret_data, schema) + _LOGGER.debug( + f"[{self.__repr__()}] Collect cloud services: " + f"{self.cloud_service_group} > {self.cloud_service_type}" + ) + response_iterator = self.collect_cloud_services( + options, secret_data, schema + ) for response in response_iterator: try: yield make_response( @@ -82,7 +92,9 @@ def collect_resources(self, options: dict, secret_data: dict, schema: str) -> Ge ) @abc.abstractmethod - def collect_cloud_services(self, options: dict, secret_data: dict, schema: str) -> Generator[dict, None, None]: + def collect_cloud_services( + self, options: dict, secret_data: dict, schema: str + ) -> Generator[dict, None, None]: raise ERROR_NOT_IMPLEMENTED() def get_cloud_service_type(self) -> dict: @@ -119,9 +131,13 @@ def collect_regions(cls, region: str = None) -> dict: def collect_metrics(self) -> dict: for dirname in os.listdir(os.path.join(METRIC_DIR, self.cloud_service_group)): - for filename in os.listdir(os.path.join(METRIC_DIR, self.cloud_service_group, dirname)): + for filename in os.listdir( + os.path.join(METRIC_DIR, self.cloud_service_group, dirname) + ): if filename.endswith(".yaml"): - file_path = os.path.join(METRIC_DIR, self.cloud_service_group, dirname, filename) + file_path = os.path.join( + METRIC_DIR, self.cloud_service_group, dirname, filename + ) info = utils.load_yaml_from_file(file_path) if filename == "namespace.yaml": yield make_response( diff --git a/src/plugin/manager/iam/group_manager.py b/src/plugin/manager/iam/group_manager.py index b5d9a74..1754c6d 100644 --- a/src/plugin/manager/iam/group_manager.py +++ b/src/plugin/manager/iam/group_manager.py @@ -79,14 +79,20 @@ def get_group_members(self, group_id: str) -> list: self.identity_connector.get_membership(member_name) ) else: - _, cached_mem_name = self.member_id_to_membership_info[member_id].get("name").split("/memberships/") + _, cached_mem_name = ( + self.member_id_to_membership_info[member_id] + .get("name") + .split("/memberships/") + ) _, curr_mem_name = member_name.split("/memberships/") if cached_mem_name != curr_mem_name: _LOGGER.debug( f"[{self.__repr__()}] MEMBERSHIP_NAME: {curr_mem_name} has different member_id: {member_id} \ from (cached) {cached_mem_name}: {self.member_id_to_membership_info[member_id]}" ) - self.member_id_to_membership_info[member_id] = self.identity_connector.get_membership(member_name) + self.member_id_to_membership_info[member_id] = ( + self.identity_connector.get_membership(member_name) + ) member_info = self.member_id_to_membership_info[member_id] if member_info.get("memberType"): member_info["memberType"] = member_info.pop("type") diff --git a/src/plugin/manager/iam/permission_manager.py b/src/plugin/manager/iam/permission_manager.py index 9adf883..784b4f3 100644 --- a/src/plugin/manager/iam/permission_manager.py +++ b/src/plugin/manager/iam/permission_manager.py @@ -30,7 +30,9 @@ def __init__(self, *args, **kwargs): "PROJECT": {}, } - def collect_cloud_services(self, options: dict, secret_data: dict, schema: str) -> Generator[dict, None, None]: + def collect_cloud_services( + self, options: dict, secret_data: dict, schema: str + ) -> Generator[dict, None, None]: self.iam_connector = IAMConnector(options, secret_data, schema) self.rm_v3_connector = ResourceManagerV3Connector(options, secret_data, schema) @@ -82,7 +84,7 @@ def collect_organization_permissions(self, organization: dict) -> None: "targetType": "ORGANIZATION", "id": organization_id, "name": organization_name, - "location": organization_name + "location": organization_name, } bindings = self.rm_v3_connector.get_organization_iam_policies(organization_id) for binding in bindings: @@ -165,10 +167,16 @@ def parse_binding_info(self, binding: dict, target: dict) -> None: if member_type == "serviceAccount": if member_id in self.service_account_info: - self.permission_info[member]["memberName"] = self.service_account_info[member_id].get("name") - self.permission_info[member]["projectId"] = self.service_account_info[member_id].get("projectId") + self.permission_info[member]["memberName"] = ( + self.service_account_info[member_id].get("name") + ) + self.permission_info[member]["projectId"] = ( + self.service_account_info[member_id].get("projectId") + ) else: - self.permission_info[member]["memberType"] = "googleManagedServiceAccount" + self.permission_info[member][ + "memberType" + ] = "googleManagedServiceAccount" if target_type == "PROJECT": self.permission_info[member]["projectId"] = target["id"] @@ -199,7 +207,9 @@ def get_folder_location(self, folder_id: str) -> str: parent = folder.get("parent") if parent.startswith("organizations/"): organization = self.rm_v3_connector.get_organization(parent) - location = f"{organization.get('displayName')} > {folder.get('displayName')}" + location = ( + f"{organization.get('displayName')} > {folder.get('displayName')}" + ) else: parent_location = self.get_folder_location(parent) location = f"{parent_location} > {folder.get('displayName')}" @@ -215,7 +225,9 @@ def get_project_location(self, project_id: str) -> str: parent = project.get("parent") if parent.startswith("organizations/"): organization = self.rm_v3_connector.get_organization(parent) - location = f"{organization.get('displayName')} > {project.get('displayName')}" + location = ( + f"{organization.get('displayName')} > {project.get('displayName')}" + ) else: parent_location = self.get_folder_location(parent) location = f"{parent_location} > {project.get('displayName')}" diff --git a/src/plugin/manager/iam/role_manager.py b/src/plugin/manager/iam/role_manager.py index 893a98b..42714fb 100644 --- a/src/plugin/manager/iam/role_manager.py +++ b/src/plugin/manager/iam/role_manager.py @@ -24,7 +24,9 @@ def __init__(self, *args, **kwargs): self.iam_connector = None self.rm_v3_connector = None - def collect_cloud_services(self, options: dict, secret_data: dict, schema: str) -> Generator[dict, None, None]: + def collect_cloud_services( + self, options: dict, secret_data: dict, schema: str + ) -> Generator[dict, None, None]: self.iam_connector = IAMConnector(options, secret_data, schema) self.rm_v3_connector = ResourceManagerV3Connector(options, secret_data, schema) default_project_id = secret_data.get("project_id") @@ -44,13 +46,17 @@ def collect_cloud_services(self, options: dict, secret_data: dict, schema: str) for project in projects: yield from self.collect_project_roles(project["projectId"]) - def collect_organization_roles(self, organization: dict, default_project_id: str) -> Generator[dict, None, None]: + def collect_organization_roles( + self, organization: dict, default_project_id: str + ) -> Generator[dict, None, None]: organization_id = organization.get("name") organization_name = organization.get("displayName") location = f"organizations/{organization_name}" roles = self.iam_connector.list_organization_roles(organization_id) for role in roles: - yield self.make_role_info(role, default_project_id, "ORGANIZATION", location) + yield self.make_role_info( + role, default_project_id, "ORGANIZATION", location + ) def collect_project_roles(self, project_id: str) -> Generator[dict, None, None]: roles = self.iam_connector.list_project_roles(project_id) @@ -58,7 +64,9 @@ def collect_project_roles(self, project_id: str) -> Generator[dict, None, None]: for role in roles: yield self.make_role_info(role, project_id, "PROJECT", location) - def make_role_info(self, role: dict, project_id: str, role_type: str, location: str = None) -> dict: + def make_role_info( + self, role: dict, project_id: str, role_type: str, location: str = None + ) -> dict: name = role.get("title") role_id = role.get("name") role_url = role_id.replace("/", "<") @@ -94,7 +102,7 @@ def make_role_info(self, role: dict, project_id: str, role_type: str, location: reference={ "resource_id": role_id, "external_link": f"https://console.cloud.google.com/iam-admin/roles/details/{role_url}?" - f"project={project_id}" + f"project={project_id}", }, # data_format="grpc", ) diff --git a/src/plugin/manager/iam/service_account_manager.py b/src/plugin/manager/iam/service_account_manager.py index c76b8bb..7d9ed76 100644 --- a/src/plugin/manager/iam/service_account_manager.py +++ b/src/plugin/manager/iam/service_account_manager.py @@ -31,10 +31,14 @@ def __init__(self, *args, **kwargs): "PROJECT": {}, } - def collect_cloud_services(self, options: dict, secret_data: dict, schema: str) -> Generator[dict, None, None]: + def collect_cloud_services( + self, options: dict, secret_data: dict, schema: str + ) -> Generator[dict, None, None]: self.iam_connector = IAMConnector(options, secret_data, schema) self.rm_v3_connector = ResourceManagerV3Connector(options, secret_data, schema) - self.logging_connector = LoggingConnector(options=options, secret_data=secret_data, schema=schema) + self.logging_connector = LoggingConnector( + options=options, secret_data=secret_data, schema=schema + ) # Get all projects projects = self.rm_v3_connector.list_all_projects() @@ -62,8 +66,14 @@ def make_cloud_service_info(self, service_account: dict, project_id: str) -> dic service_account["status"] = "DISABLED" else: service_account["status"] = "ENABLED" - service_account["lastActivityTime"] = self.logging_connector.get_last_log_entry_timestamp(project_id, email) - service_account["lastActivityDescription"] = f"Activity log found in the last {self.logging_connector.log_search_period}" if service_account["lastActivityTime"] else f"No activity log found in the last {self.logging_connector.log_search_period}" + service_account["lastActivityTime"] = ( + self.logging_connector.get_last_log_entry_timestamp(project_id, email) + ) + service_account["lastActivityDescription"] = ( + f"Activity log found in the last {self.logging_connector.log_search_period}" + if service_account["lastActivityTime"] + else f"No activity log found in the last {self.logging_connector.log_search_period}" + ) keys = self.get_service_account_keys(email, project_id) service_account["keys"] = keys service_account["keyCount"] = len(keys) @@ -79,7 +89,7 @@ def make_cloud_service_info(self, service_account: dict, project_id: str) -> dic reference={ "resource_id": resource_id, "external_link": f"https://console.cloud.google.com/iam-admin/serviceaccounts/details/{unique_id}?" - f"project={project_id}" + f"project={project_id}", }, # data_format="grpc", ) @@ -90,8 +100,16 @@ def get_service_account_keys(self, email: str, project_id: str) -> list: key_full_name = key.get("name") key["name"] = key_full_name.split("/")[-1] key["status"] = "ACTIVE" - key["lastActivityTime"] = self.logging_connector.get_last_log_entry_timestamp(project_id, email, key_full_name) - key["lastActivityDescription"] = f"Activity log found in the last {self.logging_connector.log_search_period}" if key["lastActivityTime"] else f"No activity log found in the last {self.logging_connector.log_search_period}" + key["lastActivityTime"] = ( + self.logging_connector.get_last_log_entry_timestamp( + project_id, email, key_full_name + ) + ) + key["lastActivityDescription"] = ( + f"Activity log found in the last {self.logging_connector.log_search_period}" + if key["lastActivityTime"] + else f"No activity log found in the last {self.logging_connector.log_search_period}" + ) creation_time = key.get("validAfterTime") expiration_time = key.get("validBeforeTime") diff --git a/src/plugin/utils/__init__.py b/src/plugin/utils/__init__.py index 2ac0ae4..cb84c7d 100644 --- a/src/plugin/utils/__init__.py +++ b/src/plugin/utils/__init__.py @@ -1,2 +1 @@ from plugin.utils.error_handlers import * - diff --git a/src/plugin/utils/error_handlers.py b/src/plugin/utils/error_handlers.py index 8a13a7c..48ba4af 100644 --- a/src/plugin/utils/error_handlers.py +++ b/src/plugin/utils/error_handlers.py @@ -13,16 +13,21 @@ def wrapper(self, *args, **kwargs): try: return method(self, *args, **kwargs) except Exception as e: - _LOGGER.debug(f"{self.__repr__()} Retrying {method.__name__}({args}, {kwargs}, trial={trial + 1}): {str(e)}") - sleep(1 + 0.002 ** trial) + _LOGGER.debug( + f"{self.__repr__()} Retrying {method.__name__}({args}, {kwargs}, trial={trial + 1}): {str(e)}" + ) + sleep(1 + 0.002**trial) self.client = googleapiclient.discovery.build( self.google_client_service, self.version, credentials=self.credentials, ) trial += 1 - _LOGGER.error(f"{self.__repr__()} Failed to {method.__name__}({args}, {kwargs})") + _LOGGER.error( + f"{self.__repr__()} Failed to {method.__name__}({args}, {kwargs})" + ) return default_response + return wrapper - return decorator + return decorator diff --git a/src/setup.py b/src/setup.py index 0bb3548..6e2f887 100755 --- a/src/setup.py +++ b/src/setup.py @@ -34,11 +34,6 @@ "spaceone-api", "google-api-python-client", ], - package_data={ - "plugin": [ - "metadata/*.yaml", - "metrics/**/**/*.yaml" - ] - }, + package_data={"plugin": ["metadata/*.yaml", "metrics/**/**/*.yaml"]}, zip_safe=False, )