From 9f74a66f2f6101e32da0f4ab4a5c4eec4f7a2e28 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Wed, 11 Dec 2024 21:12:43 +0900 Subject: [PATCH 1/2] feat: add history api for Asset Signed-off-by: ImMin5 --- .../collector_plugin_v1_connector.py | 114 ++++---- .../inventory_v2/interface/grpc/__init__.py | 3 + .../inventory_v2/interface/grpc/asset.py | 94 +++++++ .../inventory_v2/manager/asset_manager.py | 40 +-- .../manager/collecting_manager.py | 2 - .../inventory_v2/manager/history_manager.py | 254 ++++++++++++++++++ src/spaceone/inventory_v2/model/__init__.py | 3 +- .../inventory_v2/model/asset/database.py | 57 +++- .../inventory_v2/model/asset/request.py | 18 +- .../inventory_v2/model/asset/response.py | 39 ++- .../inventory_v2/service/asset_service.py | 86 ++++-- 11 files changed, 614 insertions(+), 96 deletions(-) create mode 100644 src/spaceone/inventory_v2/interface/grpc/asset.py create mode 100644 src/spaceone/inventory_v2/manager/history_manager.py diff --git a/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py b/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py index 8383c34..21ff8c5 100644 --- a/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py +++ b/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py @@ -49,7 +49,10 @@ def collect( yield self._convert_resource_data(resource_data) @staticmethod - def _convert_match_rule(resource_data: dict, resource_type: str) -> dict: + def _convert_match_rule_from_resource_data( + resource_data: dict, + ) -> dict: + resource_type = resource_data.get("resource_type") for rule_values in resource_data.get("match_rules", {}).values(): for index, rule_value in enumerate(rule_values): if rule_value == "cloud_service_id": @@ -67,78 +70,85 @@ def _convert_match_rule(resource_data: dict, resource_type: str) -> dict: rule_values[index] = "asset_type_id" return resource_data - def _convert_resource_data(self, resource_data: dict) -> dict: - - if "resource" in resource_data and "metadata" in resource_data["resource"]: - del resource_data["resource"]["metadata"] - - _LOGGER.debug( - f"[_convert_resource_data] before convert resource_data: {resource_data}" - ) - + @staticmethod + def _convert_resource_type_from_resource_data(resource_data: dict) -> dict: resource_type = resource_data.get("resource_type") - if resource_type == "inventory.Region": - pass if resource_type in ["inventory.CloudService", "inventory.CloudServiceType"]: if resource_type == "inventory.CloudService": resource_data["resource_type"] = "inventory.Asset" - resource_data["resource"]["asset_type_id"] = resource_data[ - "resource" - ].get("asset_type") - # resource_data["resource"]["asset_group_id"] = resource_data[ - # "resource" - # ].get("cloud_service_group") elif resource_type == "inventory.CloudServiceType": resource_data["resource_type"] = "inventory.AssetType" - resource_type = resource_data.get("resource_type") + return resource_data - # convert match rule - resource_data = self._convert_match_rule(resource_data, resource_type) + @staticmethod + def _convert_resource_in_resource_data(resource_data: dict) -> dict: + if _resource := resource_data.get("resource"): + resource_type = resource_data.get("resource_type") + if resource_type == "inventory.Region": + pass + elif resource_type == "inventory.Metric": + metric_resource_type = _resource.get("resource_type") + _resource["resource_type"] = metric_resource_type.replace( + "CloudService", "Asset" + ) - # convert keywords [instance_size, instance_type, ] - if _resource := resource_data.get("resource"): - # del _resource["metadata"] + elif resource_type == "inventory.Asset": + del _resource["metadata"] if "instance_size" in _resource: - _resource["data"]["instance_size"] = _resource.pop("instance_size") + _resource["data"]["instance_size"] = _resource.get("instance_size") if "instance_type" in _resource: - _resource["data"]["instance_type"] = _resource.pop("instance_type") + _resource["data"]["instance_type"] = _resource.get("instance_type") if "region_code" in _resource: _resource["region_id"] = ( f"{_resource['provider']}-{_resource['region_code']}" ) - if resource_type == "inventory.Asset": - asset_type_id = f"{_resource['provider']}-{_resource['cloud_service_group']}-{_resource['cloud_service_type']}" - _resource["asset_type_id"] = asset_type_id - resource_data["asset_type_id"] = asset_type_id - - if "reference" in _resource: - _resource["resource_id"] = _resource["reference"].get( - "resource_id" - ) - _resource["external_link"] = _resource["reference"].get( - "external_link" - ) - - elif resource_type == "inventory.AssetType": - - asset_type_id = f"at-{_resource['provider']}-{_resource['group']}-{_resource['name']}" - asset_groups = [ - f"ag-{_resource['provider']}-{_resource['group']}", - f"ag-{_resource['provider']}", - ] - _resource["asset_type_id"] = asset_type_id - resource_data["asset_type_id"] = asset_type_id - resource_data["asset_groups"] = asset_groups - - resource_data["icon"] = resource_data.get("tags", {}).get( - "spaceone:icon", "" + asset_type_id = f"{_resource['provider']}-{_resource['cloud_service_group']}-{_resource['cloud_service_type']}" + _resource["asset_type_id"] = asset_type_id + + if "reference" in _resource: + _resource["resource_id"] = _resource["reference"].get("resource_id") + _resource["external_link"] = _resource["reference"].get( + "external_link" ) + resource_data["asset_type_id"] = asset_type_id + + elif resource_type == "inventory.AssetType": + + asset_type_id = f"at-{_resource['provider']}-{_resource['group']}-{_resource['name']}" + asset_groups = [ + f"ag-{_resource['provider']}-{_resource['group']}", + f"ag-{_resource['provider']}", + ] + _resource["asset_type_id"] = asset_type_id + resource_data["asset_type_id"] = asset_type_id + resource_data["asset_groups"] = asset_groups + + resource_data["icon"] = resource_data.get("tags", {}).get( + "spaceone:icon", "" + ) + return resource_data + + def _convert_resource_data(self, resource_data: dict) -> dict: + + _LOGGER.debug( + f"[_convert_resource_data] before convert resource_data: {resource_data}" + ) + + # 1. convert resource_type + resource_data = self._convert_resource_type_from_resource_data(resource_data) + + # 2. convert match rule + resource_data = self._convert_match_rule_from_resource_data(resource_data) + + # 3. convert resource in resource_data + resource_data = self._convert_resource_in_resource_data(resource_data) + _LOGGER.debug(f"[_convert_resource_data] resource_data: {resource_data}") return resource_data diff --git a/src/spaceone/inventory_v2/interface/grpc/__init__.py b/src/spaceone/inventory_v2/interface/grpc/__init__.py index 3585b9f..1937288 100644 --- a/src/spaceone/inventory_v2/interface/grpc/__init__.py +++ b/src/spaceone/inventory_v2/interface/grpc/__init__.py @@ -1,4 +1,6 @@ from spaceone.core.pygrpc.server import GRPCServer + +from .asset import Asset from .region import Region from .collector import Collector from .job import Job @@ -22,3 +24,4 @@ app.add_service(Metric) app.add_service(MetricData) app.add_service(MetricExample) +app.add_service(Asset) diff --git a/src/spaceone/inventory_v2/interface/grpc/asset.py b/src/spaceone/inventory_v2/interface/grpc/asset.py new file mode 100644 index 0000000..7bf83bc --- /dev/null +++ b/src/spaceone/inventory_v2/interface/grpc/asset.py @@ -0,0 +1,94 @@ +from spaceone.api.inventory_v2.v1 import asset_pb2, asset_pb2_grpc +from spaceone.core.pygrpc import BaseAPI + +from spaceone.inventory_v2.service import AssetService + + +class Asset(BaseAPI, asset_pb2_grpc.AssetServicer): + pb2 = asset_pb2 + pb2_grpc = asset_pb2_grpc + + def create(self, request, context): + params, metadata = self.parse_request(request, context) + asset_svc = AssetService(metadata) + response: dict = asset_svc.create(params) + return self.dict_to_message(response) + + def update(self, request, context): + params, metadata = self.parse_request(request, context) + + with self.locator.get_service( + "CloudServiceService", metadata + ) as cloud_svc_service: + return self.locator.get_info( + "CloudServiceInfo", cloud_svc_service.update(params) + ) + + def pin_data(self, request, context): + params, metadata = self.parse_request(request, context) + + with self.locator.get_service( + "CloudServiceService", metadata + ) as cloud_svc_service: + return self.locator.get_info( + "CloudServiceInfo", cloud_svc_service.pin_data(params) + ) + + def delete(self, request, context): + params, metadata = self.parse_request(request, context) + + with self.locator.get_service( + "CloudServiceService", metadata + ) as cloud_svc_service: + cloud_svc_service.delete(params) + return self.locator.get_info("EmptyInfo") + + def get(self, request, context): + params, metadata = self.parse_request(request, context) + + with self.locator.get_service( + "CloudServiceService", metadata + ) as cloud_svc_service: + return self.locator.get_info( + "CloudServiceInfo", cloud_svc_service.get(params) + ) + + def list(self, request, context): + params, metadata = self.parse_request(request, context) + asset_svc = AssetService(metadata) + response: dict = asset_svc.list(params) + return self.dict_to_message(response) + + def export(self, request, context): + params, metadata = self.parse_request(request, context) + + with self.locator.get_service( + "CloudServiceService", metadata + ) as cloud_svc_service: + return self.locator.get_info("ExportInfo", cloud_svc_service.export(params)) + + def history(self, request, context): + params, metadata = self.parse_request(request, context) + asset_svc = AssetService(metadata) + response: dict = asset_svc.history(params) + return self.dict_to_message(response) + + def analyze(self, request, context): + params, metadata = self.parse_request(request, context) + + with self.locator.get_service( + "CloudServiceService", metadata + ) as cloud_svc_service: + return self.locator.get_info( + "AnalyzeInfo", cloud_svc_service.analyze(params) + ) + + def stat(self, request, context): + params, metadata = self.parse_request(request, context) + + with self.locator.get_service( + "CloudServiceService", metadata + ) as cloud_svc_service: + return self.locator.get_info( + "StatisticsInfo", cloud_svc_service.stat(params) + ) diff --git a/src/spaceone/inventory_v2/manager/asset_manager.py b/src/spaceone/inventory_v2/manager/asset_manager.py index 572920d..249ebf8 100644 --- a/src/spaceone/inventory_v2/manager/asset_manager.py +++ b/src/spaceone/inventory_v2/manager/asset_manager.py @@ -11,7 +11,7 @@ from spaceone.inventory_v2.lib.resource_manager import ResourceManager from spaceone.inventory_v2.manager.identity_manager import IdentityManager -from spaceone.inventory_v2.model.asset.database import Asset +from spaceone.inventory_v2.model.asset.database import Asset, History _LOGGER = logging.getLogger(__name__) @@ -45,6 +45,7 @@ class AssetManager(BaseManager, ResourceManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.asset_model = Asset + self.asset_history_model = History def create_asset(self, params: dict) -> Asset: def _rollback(vo: Asset): @@ -77,11 +78,11 @@ def delete_cloud_service_by_vo(asset_vo: Asset) -> None: asset_vo.delete() def get_asset( - self, - asset_id: str, - domain_id: str, - workspace_id: str = None, - user_projects: list = None, + self, + asset_id: str, + domain_id: str, + workspace_id: str = None, + user_projects: list = None, ): conditions = {"asset_id": asset_id, "domain_id": domain_id} @@ -94,12 +95,12 @@ def get_asset( return self.asset_model.get(**conditions) def list_assets( - self, - query: dict, - target: str = None, - change_filter: bool = False, - domain_id: str = None, - reference_filter: dict = None, + self, + query: dict, + target: str = None, + change_filter: bool = False, + domain_id: str = None, + reference_filter: dict = None, ) -> Tuple[QuerySet, int]: if change_filter: query = self._change_filter_tags(query) @@ -115,12 +116,12 @@ def list_assets( ) def analyze_assets( - self, - query: dict, - change_filter: bool = False, - domain_id: str = None, - reference_filter: dict = None, - ): + self, + query: dict, + change_filter: bool = False, + domain_id: str = None, + reference_filter: dict = None, + ) -> dict: if change_filter: query = self._change_filter_tags(query) query = self._change_filter_project_group_id(query, domain_id) @@ -130,6 +131,9 @@ def analyze_assets( return self.asset_model.analyze(**query, reference_filter=reference_filter) + def list_histories(self, query: dict) -> Tuple[QuerySet, int]: + return self.asset_history_model.query(**query) + def _change_filter_tags(self, query: dict) -> dict: change_filter = [] diff --git a/src/spaceone/inventory_v2/manager/collecting_manager.py b/src/spaceone/inventory_v2/manager/collecting_manager.py index c402e39..8e6b4d4 100644 --- a/src/spaceone/inventory_v2/manager/collecting_manager.py +++ b/src/spaceone/inventory_v2/manager/collecting_manager.py @@ -69,8 +69,6 @@ def collecting_resources(self, params: dict) -> bool: secret_data = params["secret_data"] plugin_info = params["plugin_info"] - print(secret_info) - if is_sub_task: _LOGGER.debug( f"[collecting_resources] start sub task: {job_task_id} " diff --git a/src/spaceone/inventory_v2/manager/history_manager.py b/src/spaceone/inventory_v2/manager/history_manager.py new file mode 100644 index 0000000..422c77d --- /dev/null +++ b/src/spaceone/inventory_v2/manager/history_manager.py @@ -0,0 +1,254 @@ +import logging +from typing import Union +from operator import itemgetter +from spaceone.core.manager import BaseManager +from spaceone.core import utils +from spaceone.inventory_v2.model.asset.database import Asset, History + +_LOGGER = logging.getLogger(__name__) + +DIFF_KEYS = [ + "name", + "ip_addresses", + "account", + "instance_type", + "instance_size", + "reference", + "region_code", + "project_id", + "data", + "tags", +] + +MAX_KEY_DEPTH = 3 + + +class HistoryManager(BaseManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.history_model = History + self.merged_data = {} + self.is_changed = False + self.collector_id = self.transaction.get_meta("collector_id") + self.job_id = self.transaction.get_meta("job_id") + self.plugin_id = self.transaction.get_meta("plugin_id") + self.secret_id = self.transaction.get_meta("secret.secret_id") + self.service_account_id = self.transaction.get_meta("secret.service_account_id") + self.user_id = self.transaction.get_meta("user_id") + + if ( + self.collector_id + and self.job_id + and self.service_account_id + and self.plugin_id + ): + self.updated_by = "COLLECTOR" + else: + self.updated_by = "USER" + + def create_history(self, params: dict) -> History: + def _rollback(vo: History): + _LOGGER.info(f"[ROLLBACK] Delete Record : {vo.history_id} ({vo.asset_id})") + vo.delete() + + history_vo: History = self.history_model.create(params) + self.transaction.add_rollback(_rollback, history_vo) + + return history_vo + + def add_new_history(self, asset_vo: Asset, new_data: dict) -> None: + self._create_history(asset_vo, new_data) + + def add_update_history( + self, asset_vo: Asset, new_data: dict, old_data: dict + ) -> None: + new_keys = new_data.keys() + + if len(set(new_keys) & set(DIFF_KEYS)) > 0: + self._create_history(asset_vo, new_data, old_data) + + def add_delete_history(self, asset_vo: Asset) -> None: + params = { + "asset_id": asset_vo.asset_id, + "domain_id": asset_vo.domain_id, + "action": "DELETE", + } + + self.create_history(params) + + def _create_history( + self, asset_vo: Asset, new_data: dict, old_data: dict = None + ) -> None: + if old_data: + action = "UPDATE" + else: + action = "CREATE" + + metadata = new_data.get("metadata", {}).get(self.plugin_id or "MANUAL", {}) + exclude_keys = metadata.get("change_history", {}).get("exclude", []) + + diff = self._make_diff(new_data, old_data, exclude_keys) + diff_count = len(diff) + + if diff_count > 0: + params = { + "asset_id": asset_vo.asset_id, + "domain_id": asset_vo.domain_id, + "action": action, + "diff": diff, + "diff_count": diff_count, + "updated_by": self.updated_by, + } + + if self.updated_by == "COLLECTOR": + params["collector_id"] = self.collector_id + params["job_id"] = self.job_id + else: + params["user_id"] = self.user_id + + self.create_history(params) + + def _make_diff(self, new_data: dict, old_data: dict, exclude_keys: list) -> list: + diff = [] + for key in DIFF_KEYS: + if key in new_data: + if old_data: + old_value = old_data.get(key) + else: + old_value = None + + diff += self._get_diff_data(key, new_data[key], old_value, exclude_keys) + + return diff + + def _get_diff_data( + self, + key: str, + new_value: any, + old_value: any, + exclude_keys: list, + depth: int = 1, + parent_key: str = None, + ) -> list: + diff = [] + + if depth == MAX_KEY_DEPTH: + if new_value != old_value: + diff_data = self._generate_diff_data( + key, parent_key, new_value, old_value, exclude_keys + ) + if diff_data: + diff.append(diff_data) + elif isinstance(new_value, dict): + if parent_key: + parent_key = f"{parent_key}.{key}" + else: + parent_key = key + + for sub_key, sub_value in new_value.items(): + if isinstance(old_value, dict): + sub_old_value = old_value.get(sub_key) + else: + sub_old_value = None + + diff += self._get_diff_data( + sub_key, + sub_value, + sub_old_value, + exclude_keys, + depth + 1, + parent_key, + ) + else: + if new_value != old_value: + diff_data = self._generate_diff_data( + key, parent_key, new_value, old_value, exclude_keys + ) + if diff_data: + diff.append(diff_data) + + return diff + + def _generate_diff_data( + self, + key: str, + parent_key: str, + new_value: any, + old_value: any, + exclude_keys: list, + ) -> Union[dict, None]: + if old_value is None: + diff_type = "ADDED" + else: + diff_type = "CHANGED" + + before = self._change_diff_value(old_value) + after = self._change_diff_value(new_value) + diff_key = key if parent_key is None else f"{parent_key}.{key}" + + if diff_key in exclude_keys: + return None + elif before == after: + return None + else: + return { + "key": diff_key, + "before": before, + "after": after, + "type": diff_type, + } + + def _change_diff_value(self, value: any) -> any: + if isinstance(value, dict): + return utils.dump_json(self._sort_dict_value(value)) + elif isinstance(value, list): + return utils.dump_json(self._sort_list_values(value)) + elif value is None: + return value + else: + return str(value) + + def _sort_dict_value(self, value: dict) -> dict: + try: + for k, v in value.items(): + if isinstance(v, dict): + value[k] = self._sort_dict_value(v) + elif isinstance(v, list): + value[k] = self._sort_list_values(v) + + return dict(sorted(value.items())) + except Exception as e: + # _LOGGER.warning( + # f"[_sort_dict_value] dict value sort error: {e}", exc_info=True + # ) + pass + + return value + + def _sort_list_values(self, values: list) -> list: + if len(values) > 0: + if isinstance(values[0], dict): + changed_list_values = [] + for value in values: + changed_list_values.append(self._sort_dict_value(value)) + + sort_keys = list(changed_list_values[0].keys()) + + if len(sort_keys) > 0: + try: + return sorted( + changed_list_values, key=itemgetter(*sort_keys[:3]) + ) + except Exception as e: + # _LOGGER.warning( + # f"[_sort_list_values] list value sort error: {e}", + # exc_info=True, + # ) + pass + + return changed_list_values + + else: + return sorted(values) + + return values diff --git a/src/spaceone/inventory_v2/model/__init__.py b/src/spaceone/inventory_v2/model/__init__.py index 6dbdd9f..e73a7c6 100644 --- a/src/spaceone/inventory_v2/model/__init__.py +++ b/src/spaceone/inventory_v2/model/__init__.py @@ -10,4 +10,5 @@ from spaceone.inventory_v2.model.metric_data.database import MetricData from spaceone.inventory_v2.model.metric_example.database import MetricExample from spaceone.inventory_v2.model.job.database import Job -from spaceone.inventory_v2.model.job_task.database import JobTask, JobTaskDetail \ No newline at end of file +from spaceone.inventory_v2.model.job_task.database import JobTask, JobTaskDetail +from spaceone.inventory_v2.model.asset.database import History diff --git a/src/spaceone/inventory_v2/model/asset/database.py b/src/spaceone/inventory_v2/model/asset/database.py index 597cdb3..b4a3f24 100644 --- a/src/spaceone/inventory_v2/model/asset/database.py +++ b/src/spaceone/inventory_v2/model/asset/database.py @@ -2,10 +2,61 @@ from datetime import datetime from spaceone.core.model.mongo_model import MongoModel -from spaceone.inventory_v2.model.asset_type.database import AssetType - from spaceone.inventory_v2.error.asset import ERROR_RESOURCE_ALREADY_DELETED -from spaceone.inventory_v2.model.region.database import Region + + +class HistoryDiff(EmbeddedDocument): + key = StringField(required=True) + before = DynamicField(default=None, null=True) + after = DynamicField(default=None, null=True) + type = StringField( + max_length=20, choices=("ADDED", "CHANGED", "DELETED"), required=True + ) + + def to_dict(self): + return dict(self.to_mongo()) + + +class History(MongoModel): + history_id = StringField(max_length=40, generate_id="history", unique=True) + asset_id = StringField(max_length=40, required=True) + action = StringField( + max_length=20, choices=("CREATE", "UPDATE", "DELETE"), required=True + ) + diff = ListField(EmbeddedDocumentField(HistoryDiff), default=[]) + diff_count = IntField(default=0) + updated_by = StringField(max_length=40, choices=("COLLECTOR", "USER")) + collector_id = StringField(max_length=40, default=None, null=True) + job_id = StringField(max_length=40, default=None, null=True) + user_id = StringField(max_length=255, default=None, null=True) + project_id = StringField(max_length=40) + workspace_id = StringField(max_length=40) + domain_id = StringField(max_length=40) + created_at = DateTimeField(auto_now=True) + + meta = { + "minimal_fields": [ + "history_id", + "action", + "diff_count", + "asset_id", + "updated_by", + "user_id", + "collector_id", + "job_id", + ], + "ordering": ["-created_at"], + "indexes": [ + { + "fields": ["domain_id", "asset_id", "-created_at", "diff.key"], + "name": "COMPOUND_INDEX_FOR_SEARCH", + }, + {"fields": ["domain_id", "history_id"], "name": "COMPOUND_INDEX_FOR_GET"}, + "collector_id", + "job_id", + "domain_id", + ], + } class Asset(MongoModel): diff --git a/src/spaceone/inventory_v2/model/asset/request.py b/src/spaceone/inventory_v2/model/asset/request.py index b90e578..cf35fb2 100644 --- a/src/spaceone/inventory_v2/model/asset/request.py +++ b/src/spaceone/inventory_v2/model/asset/request.py @@ -6,8 +6,11 @@ "AssetUpdateRequest", "AssetGetRequest", "AssetSearchQueryRequest", + "AssetHistorySearchQueryRequest", ] +Action = Literal["CREATE", "UPDATE", "DELETE"] + class AssetCreateRequest(BaseModel): asset_id: Union[str, None] @@ -53,6 +56,19 @@ class AssetGetRequest(BaseModel): class AssetSearchQueryRequest(BaseModel): query: Union[dict, None] = None - user_projects: List[str] + user_projects: Union[List[str], None] = None + workspace_id: Union[str, None] = None + domain_id: str + + +class AssetHistorySearchQueryRequest(BaseModel): + query: Union[dict, None] = None + history_id: Union[str, None] = None + asset_id: str + action: Union[Action, None] = None + user_id: Union[str, None] = None + collector_id: Union[str, None] = None + job_id: Union[str, None] = None workspace_id: Union[str, None] = None domain_id: str + user_projects: Union[List[str], None] = None diff --git a/src/spaceone/inventory_v2/model/asset/response.py b/src/spaceone/inventory_v2/model/asset/response.py index db704f0..130cd8c 100644 --- a/src/spaceone/inventory_v2/model/asset/response.py +++ b/src/spaceone/inventory_v2/model/asset/response.py @@ -1,21 +1,49 @@ from datetime import datetime -from typing import Union, List +from typing import Union, List, Literal from pydantic import BaseModel from spaceone.core import utils +__all__ = ["AssetResponse", "AssetsResponse", "AssetHistoriesResponse"] + +Action = Literal["CREATE", "UPDATE", "DELETE"] + + +class AssetHistoryResponse(BaseModel): + history_id: Union[str, None] + asset_id: Union[str, None] + action: Union[Action, None] + diff: Union[list, None] + diff_count: Union[int, None] + updated_by: Union[str, None] + user_id: Union[str, None] + collector_id: Union[str, None] + job_id: Union[str, None] + domain_id: Union[str, None] + created_at: Union[datetime, None] + + def dict(self, *args, **kwargs): + data = super().dict(*args, **kwargs) + data["created_at"] = utils.datetime_to_iso8601(data["created_at"]) + return data + + +class AssetHistoriesResponse(BaseModel): + results: List[AssetHistoryResponse] + total_count: int + class AssetResponse(BaseModel): asset_id: Union[str, None] name: Union[str, None] provider: Union[str, None] - ipaddresses: Union[List[str], None] + ip_addresses: Union[List[str], None] account: Union[str, None] data: Union[dict, None] - metadata: Union[dict, None] tags: Union[dict, None] region_id: Union[str, None] asset_type_id: Union[str, None] + secret_id: Union[str, None] service_account_id: Union[str, None] collector_id: Union[str, None] project_id: Union[str, None] @@ -29,3 +57,8 @@ def dict(self, *args, **kwargs): data["created_at"] = utils.datetime_to_iso8601(data["created_at"]) data["updated_at"] = utils.datetime_to_iso8601(data.get("updated_at")) return data + + +class AssetsResponse(BaseModel): + results: List[AssetResponse] + total_count: int diff --git a/src/spaceone/inventory_v2/service/asset_service.py b/src/spaceone/inventory_v2/service/asset_service.py index 05402a5..a3a945a 100644 --- a/src/spaceone/inventory_v2/service/asset_service.py +++ b/src/spaceone/inventory_v2/service/asset_service.py @@ -12,6 +12,7 @@ CollectionStateManager, ) from spaceone.inventory_v2.manager.collector_rule_manager import CollectorRuleManager +from spaceone.inventory_v2.manager.history_manager import HistoryManager from spaceone.inventory_v2.manager.identity_manager import IdentityManager from spaceone.inventory_v2.model.asset.database import Asset from spaceone.inventory_v2.model.asset.request import * @@ -35,7 +36,7 @@ @mutation_handler @event_handler class AssetService(BaseService): - resource = "AssetType" + resource = "Asset" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -82,7 +83,7 @@ def create(self, params: AssetCreateRequest) -> Union[AssetResponse, dict]: return AssetResponse(**asset_vo.to_dict()) def create_resource(self, params: dict) -> Asset: - # ch_mgr: ChangeHistoryManager = self.locator.get_manager("ChangeHistoryManager") + history_mgr = HistoryManager() if json_data := params.get("json_data"): params["data"] = utils.load_json(json_data) @@ -105,15 +106,9 @@ def create_resource(self, params: dict) -> Asset: del params["json_metadata"] domain_id = params["domain_id"] - workspace_id = params["workspace_id"] - - secret_project_id = self.transaction.get_meta("secret.project_id") - provider = params["provider"] - if instance_size := params.get("instance_size"): - if not isinstance(instance_size, float): - raise ERROR_INVALID_PARAMETER_TYPE(key="instance_size", type="float") + secret_project_id = self.transaction.get_meta("secret.project_id") if "tags" in params: params["tags"] = self._convert_tags_to_dict(params["tags"]) @@ -143,9 +138,8 @@ def create_resource(self, params: dict) -> Asset: asset_vo = self.asset_mgr.create_asset(params) - # todo: Create New History # Create New History - # ch_mgr.add_new_history(asset_vo, params) + history_mgr.add_new_history(asset_vo, params) # Create Collection State self.state_mgr.create_collection_state(asset_vo.asset_id, domain_id) @@ -188,7 +182,7 @@ def update(self, params: AssetUpdateRequest) -> Union[AssetResponse, dict]: @check_required(["asset_id", "workspace_id", "domain_id"]) def update_resource(self, params: dict) -> Asset: - # ch_mgr: ChangeHistoryManager = self.locator.get_manager("ChangeHistoryManager") + history_mgr = HistoryManager() if json_data := params.get("json_data"): params["data"] = utils.load_json(json_data) @@ -277,9 +271,8 @@ def update_resource(self, params: dict) -> Asset: asset_vo = self.asset_mgr.update_asset_by_vo(params, asset_vo) - # todo: Create Update History # Create Update History - # ch_mgr.add_update_history(asset_vo, params, old_asset_data) + history_mgr.add_update_history(asset_vo, params, old_asset_data) # Update Collection History state_vo = self.state_mgr.get_collection_state(asset_id, domain_id) @@ -341,7 +334,7 @@ def get(self, params: AssetGetRequest) -> Union[AssetResponse, dict]: @append_keyword_filter(_KEYWORD_FILTER) @set_query_page_limit(1000) @convert_model - def list(self, params: AssetSearchQueryRequest): + def list(self, params: AssetSearchQueryRequest) -> Union[AssetsResponse, dict]: """ Args: params (dict): { @@ -373,13 +366,74 @@ def list(self, params: AssetSearchQueryRequest): query = params.query or {} reference_filter = {"domain_id": domain_id, "workspace_id": workspace_id} - return self.asset_mgr.list_assets( + asset_vos, total_count = self.asset_mgr.list_assets( query, change_filter=True, domain_id=domain_id, reference_filter=reference_filter, ) + assets_info = [asset_vo.to_dict() for asset_vo in asset_vos] + + for asset_info in assets_info: + print(asset_info) + return AssetsResponse(results=assets_info, total_count=total_count) + + @transaction( + permission="inventory-v2:Asset.read", + role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER", "WORKSPACE_MEMBER"], + ) + @append_query_filter( + [ + "asset_id", + "history_id", + "action", + "user_id", + "collector_id", + "job_id", + "updated_by", + "domain_id", + ] + ) + @append_keyword_filter(["diff.key", "diff.before", "diff.after"]) + @convert_model + def history( + self, params: AssetHistorySearchQueryRequest + ) -> Union[AssetHistoriesResponse, dict]: + """ + Args: + params (dict): { + 'query': 'dict (spaceone.api.core.v1.Query)', + 'asset_id': 'str', # required + 'history_id': 'str', + 'action': 'str', + 'user_id': 'dict', + 'collector_id': 'str', + 'job_id': 'str', + 'updated_by': 'str', + 'workspace_id': 'str', # injected from auth + 'domain_id ': 'str', # injected from auth # required + 'user_projects': 'list', # injected from auth + } + + Returns: + results (list) + total_count (int) + """ + + self.asset_mgr.get_asset( + params.asset_id, + params.domain_id, + params.workspace_id, + params.user_projects, + ) + + query = params.query or {} + history_vos, total_count = self.asset_mgr.list_histories(query) + + histories_info = [history_vo.to_dict() for history_vo in history_vos] + return AssetHistoriesResponse(results=histories_info, total_count=total_count) + @staticmethod def _make_region_key(domain_id: str, provider: str, region_code: str) -> str: return f"{domain_id}.{provider}.{region_code}" From c1dbe92aba42bfff8ebaa10c76cf3eb2045bddd5 Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Wed, 11 Dec 2024 21:29:29 +0900 Subject: [PATCH 2/2] feat: add class variable at region manager for using ResourceManager Signed-off-by: ImMin5 --- .../collector_plugin_v1_connector.py | 5 ++++- src/spaceone/inventory_v2/manager/__init__.py | 1 - src/spaceone/inventory_v2/manager/region_manager.py | 3 +++ src/spaceone/inventory_v2/service/asset_service.py | 3 --- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py b/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py index 21ff8c5..fca82a5 100644 --- a/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py +++ b/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py @@ -87,7 +87,10 @@ def _convert_resource_in_resource_data(resource_data: dict) -> dict: if _resource := resource_data.get("resource"): resource_type = resource_data.get("resource_type") if resource_type == "inventory.Region": - pass + _resource["region_id"] = ( + f"{_resource['provider']}-{_resource['region_code']}" + ) + elif resource_type == "inventory.Metric": metric_resource_type = _resource.get("resource_type") _resource["resource_type"] = metric_resource_type.replace( diff --git a/src/spaceone/inventory_v2/manager/__init__.py b/src/spaceone/inventory_v2/manager/__init__.py index 783fdf6..083af3e 100644 --- a/src/spaceone/inventory_v2/manager/__init__.py +++ b/src/spaceone/inventory_v2/manager/__init__.py @@ -1,6 +1,5 @@ from spaceone.inventory_v2.manager.asset_manager import AssetManager from spaceone.inventory_v2.manager.asset_type_manager import AssetTypeManager -from spaceone.inventory_v2.lib.resource_manager import ResourceManager from spaceone.inventory_v2.manager.region_manager import RegionManager from spaceone.inventory_v2.manager.collecting_manager import CollectingManager from spaceone.inventory_v2.manager.namespace_group_manager import NamespaceGroupManager diff --git a/src/spaceone/inventory_v2/manager/region_manager.py b/src/spaceone/inventory_v2/manager/region_manager.py index 82de5a9..cb2bf2b 100644 --- a/src/spaceone/inventory_v2/manager/region_manager.py +++ b/src/spaceone/inventory_v2/manager/region_manager.py @@ -12,6 +12,9 @@ class RegionManager(BaseManager, ResourceManager): + resource_keys = ["region_id"] + query_method = "list_regions" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.region_model = Region() diff --git a/src/spaceone/inventory_v2/service/asset_service.py b/src/spaceone/inventory_v2/service/asset_service.py index a3a945a..86bdaa7 100644 --- a/src/spaceone/inventory_v2/service/asset_service.py +++ b/src/spaceone/inventory_v2/service/asset_service.py @@ -374,9 +374,6 @@ def list(self, params: AssetSearchQueryRequest) -> Union[AssetsResponse, dict]: ) assets_info = [asset_vo.to_dict() for asset_vo in asset_vos] - - for asset_info in assets_info: - print(asset_info) return AssetsResponse(results=assets_info, total_count=total_count) @transaction(