From 0e648f90a25e18a531d41fe87a2ae4126171d39f Mon Sep 17 00:00:00 2001 From: Jongmin Kim Date: Fri, 21 Jun 2024 22:21:17 +0900 Subject: [PATCH 1/2] feat: add metrics Signed-off-by: Jongmin Kim --- .../connector/cloud_identity_connector.py | 21 +++++ .../resource_manager_v1_connector.py | 2 +- .../resource_manager_v3_connector.py | 2 +- src/plugin/manager/base.py | 25 +++--- src/plugin/manager/iam/__init__.py | 5 +- src/plugin/manager/iam/group_manager.py | 34 ++++++++ src/plugin/manager/iam/permission_manager.py | 2 +- src/plugin/metadata/group.yaml | 80 +++++++++++++++++++ .../metrics/IAM/Permission/namespace.yaml | 8 ++ .../IAM/Permission/permission_count.yaml | 22 +++++ .../metrics/IAM/Permission/rb_count.yaml | 32 ++++++++ src/plugin/metrics/IAM/Role/namespace.yaml | 8 ++ src/plugin/metrics/IAM/Role/role_count.yaml | 20 +++++ .../metrics/IAM/ServiceAccount/namespace.yaml | 8 ++ .../metrics/IAM/ServiceAccount/sa_count.yaml | 18 +++++ .../IAM/ServiceAccount/sa_key_count.yaml | 24 ++++++ src/setup.py | 7 +- 17 files changed, 302 insertions(+), 16 deletions(-) create mode 100644 src/plugin/connector/cloud_identity_connector.py create mode 100644 src/plugin/manager/iam/group_manager.py create mode 100644 src/plugin/metadata/group.yaml create mode 100644 src/plugin/metrics/IAM/Permission/namespace.yaml create mode 100644 src/plugin/metrics/IAM/Permission/permission_count.yaml create mode 100644 src/plugin/metrics/IAM/Permission/rb_count.yaml create mode 100644 src/plugin/metrics/IAM/Role/namespace.yaml create mode 100644 src/plugin/metrics/IAM/Role/role_count.yaml create mode 100644 src/plugin/metrics/IAM/ServiceAccount/namespace.yaml create mode 100644 src/plugin/metrics/IAM/ServiceAccount/sa_count.yaml create mode 100644 src/plugin/metrics/IAM/ServiceAccount/sa_key_count.yaml diff --git a/src/plugin/connector/cloud_identity_connector.py b/src/plugin/connector/cloud_identity_connector.py new file mode 100644 index 0000000..46ba0d2 --- /dev/null +++ b/src/plugin/connector/cloud_identity_connector.py @@ -0,0 +1,21 @@ +import logging +from plugin.connector import GoogleCloudConnector + +__all__ = ["CloudIdentityConnector"] + +_LOGGER = logging.getLogger("spaceone") + + +class CloudIdentityConnector(GoogleCloudConnector): + google_client_service = "cloudidentity" + version = "v1" + + def list_groups(self, customer_id): + parent = f"customers/{customer_id}" + result = self.client.groups().list(parent=parent).execute() + print(result) + return result.get("groups", []) + + def list_memberships(self, parent): + result = self.client.groups().memberships(parent=parent).list().execute() + return result.get("memberships", []) diff --git a/src/plugin/connector/resource_manager_v1_connector.py b/src/plugin/connector/resource_manager_v1_connector.py index 03a08e4..5673ea7 100644 --- a/src/plugin/connector/resource_manager_v1_connector.py +++ b/src/plugin/connector/resource_manager_v1_connector.py @@ -4,7 +4,7 @@ __all__ = ["ResourceManagerV1Connector"] -_LOGGER = logging.getLogger(__name__) +_LOGGER = logging.getLogger("spaceone") class ResourceManagerV1Connector(GoogleCloudConnector): diff --git a/src/plugin/connector/resource_manager_v3_connector.py b/src/plugin/connector/resource_manager_v3_connector.py index ad2c890..92b6283 100644 --- a/src/plugin/connector/resource_manager_v3_connector.py +++ b/src/plugin/connector/resource_manager_v3_connector.py @@ -4,7 +4,7 @@ __all__ = ["ResourceManagerV3Connector"] -_LOGGER = logging.getLogger(__name__) +_LOGGER = logging.getLogger("spaceone") class ResourceManagerV3Connector(GoogleCloudConnector): diff --git a/src/plugin/manager/base.py b/src/plugin/manager/base.py index 5d0ce4a..628fdd9 100644 --- a/src/plugin/manager/base.py +++ b/src/plugin/manager/base.py @@ -11,7 +11,7 @@ from plugin.conf.global_conf import REGION_INFO, ICON_URL_PREFIX -_LOGGER = logging.getLogger(__name__) +_LOGGER = logging.getLogger("spaceone") CURRENT_DIR = os.path.dirname(__file__) METRIC_DIR = os.path.join(CURRENT_DIR, "../metrics/") @@ -38,10 +38,16 @@ def __repr__(self): def collect_resources(self, options: dict, secret_data: dict, schema: str) -> Generator[dict, None, None]: try: - _LOGGER.debug(f"[{self.__repr__()}.collect_resources] Collect cloud service type: {self.service}") + _LOGGER.debug(f"[{self.__repr__()}] Collect cloud service type: " + f"{self.cloud_service_group} > {self.cloud_service_type}") yield self.get_cloud_service_type() - _LOGGER.debug(f"[{self.__repr__()}.collect_resources] Collect cloud services: {self.service}") + _LOGGER.debug(f"[{self.__repr__()}] Collect metrics: " + f"{self.cloud_service_group} > {self.cloud_service_type}") + yield from self.collect_metrics() + + _LOGGER.debug(f"[{self.__repr__()}] Collect cloud services: " + f"{self.cloud_service_group} > {self.cloud_service_type}") response_iterator = self.collect_cloud_services(options, secret_data, schema) for response in response_iterator: try: @@ -58,7 +64,7 @@ def collect_resources(self, options: dict, secret_data: dict, schema: str) -> Ge ], ) except Exception as e: - _LOGGER.error(f"[{self.__repr__()}.collect_resources] Error: {str(e)}", exc_info=True) + _LOGGER.error(f"[{self.__repr__()}] Error: {str(e)}", exc_info=True) yield make_error_response( error=e, provider=self.provider, @@ -67,7 +73,7 @@ def collect_resources(self, options: dict, secret_data: dict, schema: str) -> Ge ) except Exception as e: - _LOGGER.error(f"[{self.__repr__()}.collect_resources] Error: {str(e)}", exc_info=True) + _LOGGER.error(f"[{self.__repr__()}] Error: {str(e)}", exc_info=True) yield make_error_response( error=e, provider=self.provider, @@ -111,12 +117,11 @@ def collect_regions(cls, region: str = None) -> dict: resource_type="inventory.Region", ) - @classmethod - def collect_metrics(cls, service: str) -> dict: - for dirname in os.listdir(os.path.join(METRIC_DIR, service)): - for filename in os.listdir(os.path.join(METRIC_DIR, service, dirname)): + def collect_metrics(self) -> dict: + for dirname in os.listdir(os.path.join(METRIC_DIR, self.cloud_service_group)): + for filename in os.listdir(os.path.join(METRIC_DIR, self.cloud_service_group, dirname)): if filename.endswith(".yaml"): - file_path = os.path.join(METRIC_DIR, service, dirname, filename) + file_path = os.path.join(METRIC_DIR, self.cloud_service_group, dirname, filename) info = utils.load_yaml_from_file(file_path) if filename == "namespace.yaml": yield make_response( diff --git a/src/plugin/manager/iam/__init__.py b/src/plugin/manager/iam/__init__.py index 6cc3efd..129eddd 100644 --- a/src/plugin/manager/iam/__init__.py +++ b/src/plugin/manager/iam/__init__.py @@ -1,3 +1,4 @@ from .service_account_manager import ServiceAccountManager -from .role_manager import RoleManager -from .permission_manager import PermissionManager +# from .role_manager import RoleManager +# from .permission_manager import PermissionManager +# from .group_manager import GroupManager diff --git a/src/plugin/manager/iam/group_manager.py b/src/plugin/manager/iam/group_manager.py new file mode 100644 index 0000000..8151feb --- /dev/null +++ b/src/plugin/manager/iam/group_manager.py @@ -0,0 +1,34 @@ +import logging +from typing import Generator +from spaceone.inventory.plugin.collector.lib import * +from plugin.connector.cloud_identity_connector import CloudIdentityConnector +from plugin.connector.resource_manager_v3_connector import ResourceManagerV3Connector +from plugin.manager.base import ResourceManager + +_LOGGER = logging.getLogger("spaceone") + + +class GroupManager(ResourceManager): + service = "IAM" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.cloud_service_group = "IAM" + self.cloud_service_type = "Group" + self.service_code = None + self.is_primary = False + self.icon = "iam.svg" + self.labels = [] + self.metadata_path = "metadata/group.yaml" + self.identity_connector = None + self.rm_v3_connector = None + + def collect_cloud_services(self, options: dict, secret_data: dict, schema: str) -> Generator[dict, None, None]: + self.identity_connector = CloudIdentityConnector(options, secret_data, schema) + self.rm_v3_connector = ResourceManagerV3Connector(options, secret_data, schema) + for organization in self.rm_v3_connector.search_organizations(): + print(organization) + customer_id = organization.get('directoryCustomerId') + groups = self.identity_connector.list_groups(customer_id) + print(groups) diff --git a/src/plugin/manager/iam/permission_manager.py b/src/plugin/manager/iam/permission_manager.py index 6d5c9ff..c9d1ad7 100644 --- a/src/plugin/manager/iam/permission_manager.py +++ b/src/plugin/manager/iam/permission_manager.py @@ -72,7 +72,7 @@ def make_permission_info(self) -> Generator[dict, None, None]: reference={ "resource_id": member, }, - data_format="grpc", + # data_format="grpc", ) def collect_organization_permissions(self, organization: dict) -> None: diff --git a/src/plugin/metadata/group.yaml b/src/plugin/metadata/group.yaml new file mode 100644 index 0000000..386729e --- /dev/null +++ b/src/plugin/metadata/group.yaml @@ -0,0 +1,80 @@ +search: + fields: + - Status: data.status + type: enum + enums: + - ENABLED: green.500 + label: Enabled + - DISABLED: red.500 + label: Disabled + - Role ID: data.name + - Type: data.type + type: enum + enums: + - PREDEFINED: gray.500 + label: Predefined + - CUSTOM: blue.500 + label: Custom + - Location: account + - Permission Count: data.permissionCount + data_type: integer + - Role Launch Stage: data.stage + +table: + sort: + key: data.type + fields: + - Status: data.status + type: enum + enums: + - ENABLED: green.500 + name: Enabled + type: state + - DISABLED: red.500 + name: Disabled + type: state + - Role ID: data.name + is_optional: true + - Permission Count: data.permissionCount + data_type: integer + - Type: data.type + type: enum + enums: + - PREDEFINED: gray.500 + name: Predefined + - CUSTOM: blue.500 + name: Custom + - Location: account + - Role Launch Stage: data.stage + is_optional: true + +tabs.0: + name: Details + type: item + fields: + - Role ID: data.name + - Name: name + - Status: data.status + type: enum + enums: + - ENABLED: green.500 + name: Enabled + type: state + - DISABLED: red.500 + name: Disabled + type: state + - Description: data.description + - Type: data.type + type: enum + enums: + - PREDEFINED: gray.500 + name: Predefined + - CUSTOM: blue.500 + name: Custom + - Location: account + - Role Launch Stage: data.stage + +tabs.1: + name: Permissions + type: raw + root_path: data.includedPermissions diff --git a/src/plugin/metrics/IAM/Permission/namespace.yaml b/src/plugin/metrics/IAM/Permission/namespace.yaml new file mode 100644 index 0000000..5dad626 --- /dev/null +++ b/src/plugin/metrics/IAM/Permission/namespace.yaml @@ -0,0 +1,8 @@ +--- +namespace_id: ns-google-cloud-iam-permission +name: IAM/Permission +category: ASSET +resource_type: inventory.CloudService:google_cloud.IAM.Permission +group: google_cloud +icon: 'https://spaceone-custom-assets.s3.ap-northeast-2.amazonaws.com/console-assets/icons/cloud-services/google_cloud/iam.svg' +version: '1.0' \ No newline at end of file diff --git a/src/plugin/metrics/IAM/Permission/permission_count.yaml b/src/plugin/metrics/IAM/Permission/permission_count.yaml new file mode 100644 index 0000000..6d1702d --- /dev/null +++ b/src/plugin/metrics/IAM/Permission/permission_count.yaml @@ -0,0 +1,22 @@ +--- +metric_id: metric-google-cloud-iam-permission-count +name: Permission Count +metric_type: GAUGE +resource_type: inventory.CloudService:google_cloud.IAM.Permission +query_options: + group_by: + - key: data.type + name: Permission Type + default: true + - key: account + name: Google Project ID + - key: data.status + name: Status + - key: data.inherited + name: Inherited + fields: + value: + operator: count +unit: Count +namespace_id: ns-google-cloud-iam-permission +version: '1.0' \ No newline at end of file diff --git a/src/plugin/metrics/IAM/Permission/rb_count.yaml b/src/plugin/metrics/IAM/Permission/rb_count.yaml new file mode 100644 index 0000000..f56bf40 --- /dev/null +++ b/src/plugin/metrics/IAM/Permission/rb_count.yaml @@ -0,0 +1,32 @@ +--- +metric_id: metric-google-cloud-iam-rb-count +name: Role Binding Count +metric_type: GAUGE +resource_type: inventory.CloudService:google_cloud.IAM.Permission +query_options: + unwind: + path: data.bindings + group_by: + - key: data.type + name: Principal Type + default: true + - key: data.memberId + name: Principal ID + - key: data.bindings.role.id + name: Role ID + - key: data.bindings.role.name + name: Role Name + - key: data.bindings.role.roleType + name: Role Type + - key: data.bindings.target.type + name: Target Type + - key: data.bindings.target.location + name: Location + - key: data.bindings.target.id + name: Target ID + fields: + value: + operator: count +unit: Count +namespace_id: ns-google-cloud-iam-permission +version: '1.0' \ No newline at end of file diff --git a/src/plugin/metrics/IAM/Role/namespace.yaml b/src/plugin/metrics/IAM/Role/namespace.yaml new file mode 100644 index 0000000..166defe --- /dev/null +++ b/src/plugin/metrics/IAM/Role/namespace.yaml @@ -0,0 +1,8 @@ +--- +namespace_id: ns-google-cloud-iam-role +name: IAM/Role +category: ASSET +resource_type: inventory.CloudService:google_cloud.IAM.Role +group: google_cloud +icon: 'https://spaceone-custom-assets.s3.ap-northeast-2.amazonaws.com/console-assets/icons/cloud-services/google_cloud/iam.svg' +version: '1.0' \ No newline at end of file diff --git a/src/plugin/metrics/IAM/Role/role_count.yaml b/src/plugin/metrics/IAM/Role/role_count.yaml new file mode 100644 index 0000000..742b027 --- /dev/null +++ b/src/plugin/metrics/IAM/Role/role_count.yaml @@ -0,0 +1,20 @@ +--- +metric_id: metric-google-cloud-iam-role-count +name: Role Count +metric_type: GAUGE +resource_type: inventory.CloudService:google_cloud.IAM.Role +query_options: + group_by: + - key: data.type + name: Role Type + default: true + - key: account + name: Location + - key: data.status + name: Status + fields: + value: + operator: count +unit: Count +namespace_id: ns-google-cloud-iam-role +version: '1.0' \ No newline at end of file diff --git a/src/plugin/metrics/IAM/ServiceAccount/namespace.yaml b/src/plugin/metrics/IAM/ServiceAccount/namespace.yaml new file mode 100644 index 0000000..1f858a2 --- /dev/null +++ b/src/plugin/metrics/IAM/ServiceAccount/namespace.yaml @@ -0,0 +1,8 @@ +--- +namespace_id: ns-google-cloud-iam-sa +name: IAM/ServiceAccount +category: ASSET +resource_type: inventory.CloudService:google_cloud.IAM.ServiceAccount +group: google_cloud +icon: 'https://spaceone-custom-assets.s3.ap-northeast-2.amazonaws.com/console-assets/icons/cloud-services/google_cloud/iam.svg' +version: '1.0' \ No newline at end of file diff --git a/src/plugin/metrics/IAM/ServiceAccount/sa_count.yaml b/src/plugin/metrics/IAM/ServiceAccount/sa_count.yaml new file mode 100644 index 0000000..5dbe3e8 --- /dev/null +++ b/src/plugin/metrics/IAM/ServiceAccount/sa_count.yaml @@ -0,0 +1,18 @@ +--- +metric_id: metric-google-cloud-iam-sa-count +name: Service Account Count +metric_type: GAUGE +resource_type: inventory.CloudService:google_cloud.IAM.ServiceAccount +query_options: + group_by: + - key: data.projectId + name: Google Project ID + default: true + - key: data.status + name: Status + fields: + value: + operator: count +unit: Count +namespace_id: ns-google-cloud-iam-sa +version: '1.0' \ No newline at end of file diff --git a/src/plugin/metrics/IAM/ServiceAccount/sa_key_count.yaml b/src/plugin/metrics/IAM/ServiceAccount/sa_key_count.yaml new file mode 100644 index 0000000..f20fe7b --- /dev/null +++ b/src/plugin/metrics/IAM/ServiceAccount/sa_key_count.yaml @@ -0,0 +1,24 @@ +--- +metric_id: metric-google-cloud-iam-sa-key-count +name: Key Count +metric_type: GAUGE +resource_type: inventory.CloudService:google_cloud.IAM.ServiceAccount +query_options: + unwind: + path: data.keys + group_by: + - key: data.projectId + name: Google Project ID + default: true + - key: data.keys.status + name: Status + - key: data.keys.keyType + name: Key Type + - key: data.keys.keyAlgorithm + name: Key Algorithm + fields: + value: + operator: count +unit: Count +namespace_id: ns-google-cloud-iam-sa +version: '1.0' \ No newline at end of file diff --git a/src/setup.py b/src/setup.py index c488eb9..0bb3548 100755 --- a/src/setup.py +++ b/src/setup.py @@ -34,6 +34,11 @@ "spaceone-api", "google-api-python-client", ], - package_data={"plugin": ["metadata/*.yaml"]}, + package_data={ + "plugin": [ + "metadata/*.yaml", + "metrics/**/**/*.yaml" + ] + }, zip_safe=False, ) From 96256150e44779083b09eae29c05fc2134bece95 Mon Sep 17 00:00:00 2001 From: Jongmin Kim Date: Fri, 21 Jun 2024 22:23:34 +0900 Subject: [PATCH 2/2] fix: add missing imports Signed-off-by: Jongmin Kim --- src/plugin/manager/iam/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugin/manager/iam/__init__.py b/src/plugin/manager/iam/__init__.py index 129eddd..efe87ce 100644 --- a/src/plugin/manager/iam/__init__.py +++ b/src/plugin/manager/iam/__init__.py @@ -1,4 +1,4 @@ from .service_account_manager import ServiceAccountManager -# from .role_manager import RoleManager -# from .permission_manager import PermissionManager +from .role_manager import RoleManager +from .permission_manager import PermissionManager # from .group_manager import GroupManager