From 7352f949135768e0f641745f1068a5304c9eee50 Mon Sep 17 00:00:00 2001 From: Jongmin Kim Date: Wed, 8 Nov 2023 19:49:53 +0900 Subject: [PATCH] feat: add cost_data_keys to DataSource model --- .../cost_analysis/info/data_source_info.py | 1 + .../cost_analysis/model/data_source_model.py | 8 ++++---- .../cost_analysis/service/job_service.py | 18 +++++++++++++++--- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/spaceone/cost_analysis/info/data_source_info.py b/src/spaceone/cost_analysis/info/data_source_info.py index 2bdd0f7c..fb00aade 100644 --- a/src/spaceone/cost_analysis/info/data_source_info.py +++ b/src/spaceone/cost_analysis/info/data_source_info.py @@ -55,6 +55,7 @@ def DataSourceInfo(data_source_vo: DataSource, minimal=False): 'tags': change_struct_type(data_source_vo.tags), 'cost_tag_keys': data_source_vo.cost_tag_keys, 'cost_additional_info_keys': data_source_vo.cost_additional_info_keys, + 'cost_data_keys': data_source_vo.cost_data_keys, 'domain_id': data_source_vo.domain_id, 'created_at': utils.datetime_to_iso8601(data_source_vo.created_at), 'last_synchronized_at': utils.datetime_to_iso8601(data_source_vo.last_synchronized_at) diff --git a/src/spaceone/cost_analysis/model/data_source_model.py b/src/spaceone/cost_analysis/model/data_source_model.py index 942725df..afa5990c 100644 --- a/src/spaceone/cost_analysis/model/data_source_model.py +++ b/src/spaceone/cost_analysis/model/data_source_model.py @@ -39,6 +39,7 @@ class DataSource(MongoModel): tags = DictField(default={}) cost_tag_keys = ListField(StringField()) cost_additional_info_keys = ListField(StringField()) + cost_data_keys = ListField(StringField()) domain_id = StringField(max_length=40) created_at = DateTimeField(auto_now_add=True) last_synchronized_at = DateTimeField(default=None, null=True) @@ -53,7 +54,8 @@ class DataSource(MongoModel): 'tags', 'last_synchronized_at', 'cost_tag_keys', - 'cost_additional_info_keys' + 'cost_additional_info_keys', + 'cost_data_keys' ], 'minimal_fields': [ 'data_source_id', @@ -71,8 +73,6 @@ class DataSource(MongoModel): 'state', 'data_source_type', 'provider', - 'domain_id', - 'cost_tag_keys', - 'cost_additional_info_keys', + 'domain_id' ] } diff --git a/src/spaceone/cost_analysis/service/job_service.py b/src/spaceone/cost_analysis/service/job_service.py index dc17b809..4b10bdec 100644 --- a/src/spaceone/cost_analysis/service/job_service.py +++ b/src/spaceone/cost_analysis/service/job_service.py @@ -186,6 +186,7 @@ def get_cost_data(self, params): schema = plugin_info.get('schema') tag_keys = data_source_vo.cost_tag_keys additional_info_keys = data_source_vo.cost_additional_info_keys + data_keys = data_source_vo.cost_data_keys secret_type = data_source_vo.secret_type options.update({'secret_type': secret_type}) @@ -216,6 +217,7 @@ def get_cost_data(self, params): tag_keys = self._append_tag_keys(tag_keys, cost_data) additional_info_keys = self._append_additional_info_keys(additional_info_keys, cost_data) + data_keys = self._append_data_keys(data_keys, cost_data) if self._is_job_canceled(job_id, domain_id): self.job_task_mgr.change_canceled_status(job_task_vo) @@ -229,7 +231,7 @@ def get_cost_data(self, params): _LOGGER.debug(f'[get_cost_data] end job ({job_task_id}): {end_dt}') _LOGGER.debug(f'[get_cost_data] total job time ({job_task_id}): {end_dt - start_dt}') - self._update_tag_and_additional_info_keys(data_source_vo, tag_keys, additional_info_keys) + self._update_keys(data_source_vo, tag_keys, additional_info_keys, data_keys) self.job_task_mgr.change_success_status(job_task_vo, count) except Exception as e: @@ -387,6 +389,15 @@ def _append_additional_info_keys(additional_info_keys, cost_data): additional_info_keys.append(key) return additional_info_keys + @staticmethod + def _append_data_keys(data_keys, cost_data): + cost_data_info = cost_data.get('data') or {} + + for key in cost_data_info.keys(): + if key not in data_keys: + data_keys.append(key) + return data_keys + def _get_secret_data(self, secret_id, domain_id): secret_mgr: SecretManager = self.locator.get_manager('SecretManager') if secret_id: @@ -474,10 +485,11 @@ def _close_job(self, job_id, domain_id, data_source_id): elif job_vo.status == 'CANCELED': self._rollback_cost_data(job_vo) - def _update_tag_and_additional_info_keys(self, data_source_vo, tag_keys, additional_info_keys): + def _update_keys(self, data_source_vo, tag_keys, additional_info_keys, data_keys): self.data_source_mgr.update_data_source_by_vo({ 'cost_tag_keys': tag_keys, - 'cost_additional_info_keys': additional_info_keys + 'cost_additional_info_keys': additional_info_keys, + 'cost_data_keys': data_keys }, data_source_vo) def _rollback_cost_data(self, job_vo: Job):