diff --git a/src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py b/src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py index 4d063ca..5b43c53 100644 --- a/src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py +++ b/src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py @@ -106,7 +106,7 @@ def get_cost_data(self, blobs): for blob in blobs: cost_csv = self._download_cost_data(blob) - df = pd.read_csv(StringIO(cost_csv)) + df = pd.read_csv(StringIO(cost_csv), low_memory=False) df = df.replace({np.nan: None}) costs_data = df.to_dict('records') diff --git a/src/cloudforet/cost_analysis/manager/cost_manager.py b/src/cloudforet/cost_analysis/manager/cost_manager.py index 0583ef4..3db2dfa 100644 --- a/src/cloudforet/cost_analysis/manager/cost_manager.py +++ b/src/cloudforet/cost_analysis/manager/cost_manager.py @@ -32,16 +32,16 @@ def get_data(self, options, secret_data, schema, task_options): parameters = self._make_parameters(_start, _end, options) start_time = time.time() - print(f"{datetime.utcnow()} [INFO][get_data] start to collect data from {_start} to {_end}") + _LOGGER.info(f'[get_data] start to collect data from {_start} to {_end}') for idx, tenant_id in enumerate(tenant_ids): scope = self._make_scope(secret_data, task_options, collect_scope, tenant_id) blobs = self.azure_cm_connector.begin_create_operation(scope, parameters) response_stream = self.azure_cm_connector.get_cost_data(blobs) for results in response_stream: yield self._make_cost_data(results=results, end=_end, tenant_id=tenant_id, options=options) - print(f"{datetime.utcnow()} [INFO][get_data] #{idx+1} {tenant_id} tenant collect is done") + _LOGGER.info(f'[get_data] #{idx+1} {tenant_id} tenant collect is done') end_time = time.time() - print(f"{datetime.utcnow()} [INFO][get_data] all collect is done in {int(end_time - start_time)} seconds") + _LOGGER.info(f'[get_data] all collect is done in {int(end_time - start_time)} seconds') yield [] def _make_cost_data(self, results, end, options, tenant_id=None): @@ -102,9 +102,6 @@ def _get_additional_info(self, result, options, tenant_id=None): if meter_category == 'Virtual Machines' and 'Meter' in result: additional_info['Instance Type'] = result['meter'] - if result.get('resourcelocation') != '' and result.get('resourcelocation'): - additional_info['Resource Location'] = result['resourcelocation'] - if result.get('resourcegroupname') != '' and result.get('resourcegroupname'): additional_info['Resource Group'] = result['resourcegroupname'] @@ -185,13 +182,20 @@ def _make_scope(secret_data, task_options, collect_scope, customer_tenant_id=Non return scope @staticmethod - def _convert_tags_str_to_dict(tags: str): + def _convert_tags_str_to_dict(tags_str: str): try: - if tags is None: + if tags_str is None: return {} - return json.loads(tags) + + if tags_str[0] != '{' and tags_str[:-1] != '}': + tags_str = '{' + tags_str + '}' + + tags = json.loads(tags_str) + return tags except Exception as e: - _LOGGER.error(f'[_convert_tags_str_to_dict] tags : {tags} {e}') + print(type(tags_str)) + print(tags_str) + _LOGGER.error(f'[_convert_tags_str_to_dict] tags : {tags_str} {e}') return {} @staticmethod