From 2cb510bad74d6472d843fc95c12b6cd79914a7bb Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 2 Oct 2024 09:33:12 +0000 Subject: [PATCH] feat(metrics-type): Add metrics type --- app/exporter.py | 7 +++-- exporter_config.yaml | 75 +++++++++++++++++++++++--------------------- main.py | 35 ++++++++++++++++++++- 3 files changed, 78 insertions(+), 39 deletions(-) diff --git a/app/exporter.py b/app/exporter.py index 2cc966c..8c6051b 100644 --- a/app/exporter.py +++ b/app/exporter.py @@ -20,6 +20,7 @@ def __init__( aws_assumed_role_name, group_by, targets, + metrics_type, ): self.polling_interval_seconds = polling_interval_seconds self.metric_name = metric_name @@ -28,6 +29,7 @@ def __init__( self.aws_access_secret = aws_access_secret self.aws_assumed_role_name = aws_assumed_role_name self.group_by = group_by + self.metrics_type = metrics_type # Store metrics # we have verified that there is at least one target self.labels = set(targets[0].keys()) # for now we only support exporting one type of cost (Usage) @@ -40,7 +42,7 @@ def __init__( def run_metrics(self): # every time we clear up all the existing labels before setting new ones self.aws_daily_cost_usd .clear() - + for aws_account in self.targets: logging.info("querying cost data for aws account %s" % aws_account["Publisher"]) try: @@ -85,9 +87,10 @@ def query_aws_cost_explorer(self, aws_client, group_by): TimePeriod={"Start": start_date.strftime("%Y-%m-%d"), "End": end_date.strftime("%Y-%m-%d")}, Filter={"Dimensions": {"Key": "RECORD_TYPE", "Values": ["Usage"]}}, Granularity="DAILY", - Metrics=["UnblendedCost"], + Metrics=self.metrics_type, # Use dynamic metrics GroupBy=groups, ) + return response["ResultsByTime"] def fetch(self, aws_account): diff --git a/exporter_config.yaml b/exporter_config.yaml index e443319..fe04e89 100644 --- a/exporter_config.yaml +++ b/exporter_config.yaml @@ -6,42 +6,45 @@ aws_access_secret: $AWS_ACCESS_SECRET|"" # for prod deployment, DO NOT put the a aws_assumed_role_name: example-assumerole metrics: -- metric_name: aws_daily_cost_by_service_by_account # change the metric name if needed - group_by: - enabled: true - # Cost data can be groupped using up to two different groups: DIMENSION, TAG, COST_CATEGORY. - # ref: https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_GetCostAndUsageWithResources.html - # note: label_name should be unique, and different from the labes in target_aws_accounts - groups: - - type: DIMENSION - key: SERVICE - label_name: ServiceName - - type: DIMENSION - key: LINKED_ACCOUNT - label_name: AccountName - merge_minor_cost: - # if this is enabled, minor cost that is below the threshold will be merged into one group - enabled: false - threshold: 10 - tag_value: other -- metric_name: aws_daily_cost_usd # change the metric name if needed - group_by: - enabled: false - # Cost data can be groupped using up to two different groups: DIMENSION, TAG, COST_CATEGORY. - # ref: https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_GetCostAndUsageWithResources.html - # note: label_name should be unique, and different from the labes in target_aws_accounts - groups: - - type: DIMENSION - key: SERVICE - label_name: ServiceName - - type: DIMENSION - key: REGION - label_name: RegionName - merge_minor_cost: - # if this is enabled, minor cost that is below the threshold will be merged into one group - enabled: false - threshold: 10 - tag_value: other + - metric_name: aws_daily_cost_by_service_by_account # change the metric name if needed + group_by: + enabled: true + # Cost data can be groupped using up to two different groups: DIMENSION, TAG, COST_CATEGORY. + # ref: https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_GetCostAndUsageWithResources.html + # note: label_name should be unique, and different from the labes in target_aws_accounts + groups: + - type: DIMENSION + key: SERVICE + label_name: ServiceName + - type: DIMENSION + key: LINKED_ACCOUNT + label_name: AccountName + merge_minor_cost: + # if this is enabled, minor cost that is below the threshold will be merged into one group + enabled: false + threshold: 10 + tag_value: other + # Allowed values for metric type are AmortizedCost, BlendedCost, NetAmortizedCost, NetUnblendedCost, NormalizedUsageAmount, UnblendedCost, and UsageQuantity + metrics_type: AmortizedCost + + - metric_name: aws_daily_cost_usd # change the metric name if needed + group_by: + # Cost data can be groupped using up to two different groups: DIMENSION, TAG, COST_CATEGORY. + # ref: https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_GetCostAndUsageWithResources.html + # note: label_name should be unique, and different from the labes in target_aws_accounts + groups: + - type: DIMENSION + key: SERVICE + label_name: ServiceName + - type: DIMENSION + key: REGION + label_name: RegionName + merge_minor_cost: + # if this is enabled, minor cost that is below the threshold will be merged into one group + enabled: false + threshold: 10 + tag_value: other + metrics_type: AmortizedCost target_aws_accounts: # here defines a list of target AWS accounts diff --git a/main.py b/main.py index 410201a..8e57daf 100644 --- a/main.py +++ b/main.py @@ -34,6 +34,11 @@ def get_configs(): def validate_configs(config): + valid_metrics_types = [ + "AmortizedCost", "BlendedCost", "NetAmortizedCost", "NetUnblendedCost", + "NormalizedUsageAmount", "UnblendedCost", "UsageQuantity" + ] + if len(config["target_aws_accounts"]) == 0: logging.error("There should be at least one target AWS accounts defined in the config!") sys.exit(1) @@ -66,10 +71,36 @@ def validate_configs(config): logging.error("Some label names in group_by are the same as AWS account labels!") sys.exit(1) + # Validate metrics_type + if config_metric["metrics_type"] not in valid_metrics_types: + logging.error(f"Invalid metrics_type: {config_metric['metrics_type']}. It must be one of {', '.join(valid_metrics_types)}.") + sys.exit(1) + for i in range(1, len(config["target_aws_accounts"])): + if labels != config["target_aws_accounts"][i].keys(): + logging.error("All the target AWS accounts should have the same set of keys (labels)!") + sys.exit(1) -def main(config): + for config_metric in config["metrics"]: + if config_metric["group_by"]["enabled"]: + if len(config_metric["group_by"]["groups"]) < 1 or len(config_metric["group_by"]["groups"]) > 2: + logging.error("If group_by is enabled, there should be at least one group, and at most two groups!") + sys.exit(1) + group_label_names = set() + for group in config_metric["group_by"]["groups"]: + if group["label_name"] in group_label_names: + logging.error("Group label names should be unique!") + sys.exit(1) + else: + group_label_names.add(group["label_name"]) + if group_label_names and (group_label_names & set(labels)): + logging.error("Some label names in group_by are the same as AWS account labels!") + sys.exit(1) + + + +def main(config): start_http_server(config["exporter_port"]) while True: for config_metric in config["metrics"]: @@ -81,10 +112,12 @@ def main(config): targets=config["target_aws_accounts"], metric_name=config_metric["metric_name"], group_by=config_metric["group_by"], + metrics_type=[config_metric["metrics_type"]], ) app_metrics.run_metrics() time.sleep(config["polling_interval_seconds"]) + if __name__ == "__main__": logger_format = "%(asctime)-15s %(levelname)-8s %(message)s" logging.basicConfig(level=logging.INFO, format=logger_format)