From 85b233583fb1963c2e1eb2d5a6c0a1976c0fd576 Mon Sep 17 00:00:00 2001 From: Guslington Date: Wed, 25 May 2022 19:59:33 +1000 Subject: [PATCH 1/6] ability to tag alarms --- lib/cfnguardian.rb | 28 ++++++++++++- lib/cfnguardian/cloudwatch.rb | 4 ++ lib/cfnguardian/compile.rb | 3 +- lib/cfnguardian/models/alarm.rb | 5 ++- lib/cfnguardian/tagger.rb | 71 +++++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 lib/cfnguardian/tagger.rb diff --git a/lib/cfnguardian.rb b/lib/cfnguardian.rb index edbb2be..93ad74a 100644 --- a/lib/cfnguardian.rb +++ b/lib/cfnguardian.rb @@ -11,6 +11,7 @@ require "cfnguardian/drift" require "cfnguardian/codecommit" require "cfnguardian/codepipeline" +require "cfnguardian/tagger" module CfnGuardian class Cli < Thor @@ -117,14 +118,37 @@ def deploy deployer.execute_change_set(change_set.id) deployer.wait_for_execute(change_set_type) end - + + desc "tag-alarms", "apply tags to the cloudwatch alarms deployed" + long_desc <<-LONG + Because Cloudformation desn't support tagging cloudwatch alarms this command + applies tags to each cloudwatch alarm created by guardian. + Guardian defines default tags and this can be added to through the alarms.yaml config. + LONG + method_option :config, aliases: :c, type: :string, desc: "yaml config file", required: true + method_option :region, aliases: :r, type: :string, desc: "set the AWS region" + + def tag_alarms + set_log_level(options[:debug]) + set_region(options[:region],true) + + compiler = CfnGuardian::Compile.new(options[:config]) + compiler.get_resources + alarms = compiler.alarms + + tagger = CfnGuardian::Tagger.new() + alarms.each do |alarm| + tagger.tag_alarm(alarm, compiler.global_tags) + end + end + desc "show-drift", "Cloudformation drift detection" long_desc <<-LONG Displays any cloudformation drift detection in the cloudwatch alarms from the deployed stacks LONG method_option :stack_name, aliases: :s, type: :string, default: 'guardian', desc: "set the Cloudformation stack name" method_option :region, aliases: :r, type: :string, desc: "set the AWS region" - + def show_drift set_region(options[:region],true) diff --git a/lib/cfnguardian/cloudwatch.rb b/lib/cfnguardian/cloudwatch.rb index 10ae791..1f5367d 100644 --- a/lib/cfnguardian/cloudwatch.rb +++ b/lib/cfnguardian/cloudwatch.rb @@ -9,6 +9,10 @@ def self.get_alarm_name(alarm) alarm_id = alarm.resource_name.nil? ? alarm.resource_id : alarm.resource_name return "guardian-#{alarm.group}-#{alarm_id}-#{alarm.name}" end + + def self.get_alarm_arn(alarm) + return "arn:aws:cloudwatch:#{Aws.config[:region]}:#{aws_account_id()}:alarm:#{self.get_alarm_name(alarm)}" + end def self.get_alarms_by_prefix(prefix:, state: nil, action_prefix: nil) client = Aws::CloudWatch::Client.new() diff --git a/lib/cfnguardian/compile.rb b/lib/cfnguardian/compile.rb index 01b3a89..682d92c 100644 --- a/lib/cfnguardian/compile.rb +++ b/lib/cfnguardian/compile.rb @@ -57,7 +57,7 @@ module CfnGuardian class Compile include Logging - attr_reader :cost, :resources, :topics + attr_reader :cost, :resources, :topics, :global_tags def initialize(config_file) config = YAML.load_file(config_file) @@ -68,6 +68,7 @@ def initialize(config_file) @topics = config.fetch('Topics',{}) @maintenance_groups = config.fetch('MaintenanceGroups', {}) @event_subscriptions = config.fetch('EventSubscriptions', {}) + @global_tags = config.fetch('GlobalTags', {}) # Make sure the default topics exist if they aren't supplied in the alarms.yaml %w(Critical Warning Task Informational Events).each do |topic| diff --git a/lib/cfnguardian/models/alarm.rb b/lib/cfnguardian/models/alarm.rb index e037304..bb13fa7 100644 --- a/lib/cfnguardian/models/alarm.rb +++ b/lib/cfnguardian/models/alarm.rb @@ -29,7 +29,8 @@ class BaseAlarm :evaluate_low_sample_count_percentile, :unit, :maintenance_groups, - :additional_notifiers + :additional_notifiers, + :tags def initialize(resource) @type = 'Alarm' @@ -56,6 +57,7 @@ def initialize(resource) @treat_missing_data = nil @maintenance_groups = [] @additional_notifiers = [] + @tags = {} end def metric_name=(metric_name) @@ -64,7 +66,6 @@ def metric_name=(metric_name) end end - class ApiGatewayAlarm < BaseAlarm def initialize(resource) super(resource) diff --git a/lib/cfnguardian/tagger.rb b/lib/cfnguardian/tagger.rb new file mode 100644 index 0000000..433c121 --- /dev/null +++ b/lib/cfnguardian/tagger.rb @@ -0,0 +1,71 @@ +require 'aws-sdk-cloudwatch' +require 'cfnguardian/cloudwatch' +require 'cfnguardian/log' + +module CfnGuardian + class Tagger + include Logging + + def initialize() + @client = Aws::CloudWatch::Client.new() + end + + def tag_alarm(alarm, global_tags={}) + alarm_arn = CfnGuardian::CloudWatch.get_alarm_arn(alarm) + + new_tags = get_tags(alarm, global_tags) + current_tags = get_alarm_tags(alarm_arn) + + tags_to_delete = get_tags_to_delete(current_tags, new_tags) + + if tags_to_delete.any? + logger.debug "Removing tags #{tags_to_delete} from alarm #{alarm_arn}" + @client.untag_resource({ + resource_arn: alarm_arn, + tag_keys: tags_to_delete + }) + end + + if tags_changed?(current_tags, new_tags) + # Aws::CloudWatch::Errors::Throttling + logger.debug "Updating tags on alarm #{alarm_arn}" + @client.tag_resource({ + resource_arn: alarm_arn, + tags: new_tags.map {|key,value| {key: key, value: value}} + }) + end + end + + def get_tags(alarm, global_tags) + defaults = { + 'guardian:resource:id': alarm.resource_id, + 'guardian:resource:group': alarm.group, + 'guardian:alarm:name': alarm.name, + 'guardian:alarm:metric': alarm.metric_name, + 'guardian:alarm:severity': alarm.alarm_action + } + tags = global_tags.merge(defaults) + return alarm.tags.merge(tags) + end + + def get_alarm_tags(alarm_arn) + resp = @client.list_tags_for_resource({ + resource_arn: alarm_arn + }) + return resp.tags + end + + def get_tags_to_delete(current_tags, new_tags) + return current_tags.select {|tag| !new_tags.has_key?(tag.key.to_sym)}.map {|tag| tag.key} + end + + def tags_changed?(current_tags, new_tags) + return tags_to_hash(current_tags) != new_tags + end + + def tags_to_hash(tags) + return tags.map {|tag| {tag.key.to_sym => tag.value} }.reduce(Hash.new, :merge) + end + + end +end \ No newline at end of file From 2099641a8eb427f0b048847cc43d66203adbe52a Mon Sep 17 00:00:00 2001 From: Guslington Date: Wed, 25 May 2022 20:00:59 +1000 Subject: [PATCH 2/6] document alarm tags --- docs/alarm_tags.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++ docs/overview.md | 3 ++- 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 docs/alarm_tags.md diff --git a/docs/alarm_tags.md b/docs/alarm_tags.md new file mode 100644 index 0000000..6e0b269 --- /dev/null +++ b/docs/alarm_tags.md @@ -0,0 +1,63 @@ +# Guardian Alarm Tags + +AWS tags can be applied to Cloudwatch alarms created by guardian. This is available as a separate guardian alarm because Cloudformation doesn't support creating tags on Cloudwatch alarms. + +## Default Tags + +Guardian will add the following default tags to each alarm + +``` +guardian:resource:id +guardian:resource:group +guardian:alarm:name +guardian:alarm:metric +guardian:alarm:severity +``` + +## Adding Tags + +Additional tags can added through the alarms yaml configuration file. They can be applied globally to all alarms, to all alarms in a resource group or a specific alarm. + +### Global Tags + +Global tags are applied to every alarm created by guardian. Add the `GlobalTags` key at the top level of the alarms yaml config with key:value pairs. + +```yml +GlobalTags: + key: value + env: production +``` + +### Resource Group Tags + +Resource group tags are applied to every alarm in a guardian resource group using the `Templates` section to add the tags. + +```yaml +Templates: + Ec2Instance: + GroupOverrides: + Tags: + key: value + env: production +``` + +### Specific Alarm Tags + +To add tags to a specific guardian alarm you can apply the tags in the `Templates` section of the alarms yaml config. + +```yaml +Templates: + Ec2Instance: + CPUUtilizationHigh: + Tags: + key: value + alarm-action: restart ec2 instance +``` + +## Applying tags + +To apply the tags run the `tag-alarms` command passing the alarms yaml config. + +```sh +cfn-guardian tag-alarms --config alarms.yaml +``` \ No newline at end of file diff --git a/docs/overview.md b/docs/overview.md index eaf736f..1342fcb 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -20,4 +20,5 @@ 7. [Maintenance Mode](maintenance_mode.md) 8. [Composite Alarms](composite_alarms.md) 9. [Alarms for Custom Metrics](custom_metrics.md) -10. [Dimension Variables](variables.md) \ No newline at end of file +10. [Dimension Variables](variables.md) +10. [Alarm Tags](alarm_tags.md) \ No newline at end of file From 283cb471fcc8c2795ad004fd548826c852b4ba21 Mon Sep 17 00:00:00 2001 From: Guslington Date: Thu, 26 May 2022 10:26:36 +1000 Subject: [PATCH 3/6] fixup doco --- docs/alarm_tags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/alarm_tags.md b/docs/alarm_tags.md index 6e0b269..f4c4600 100644 --- a/docs/alarm_tags.md +++ b/docs/alarm_tags.md @@ -1,6 +1,6 @@ # Guardian Alarm Tags -AWS tags can be applied to Cloudwatch alarms created by guardian. This is available as a separate guardian alarm because Cloudformation doesn't support creating tags on Cloudwatch alarms. +AWS tags can be applied to Cloudwatch alarms created by guardian. This is available as a separate guardian command [`cfn-guardian tag-alarms`] because Cloudformation doesn't support creating tags on Cloudwatch alarms. ## Default Tags From 8d32481816f55339bcf3e49965ef5dac1ee3cdce Mon Sep 17 00:00:00 2001 From: Guslington Date: Thu, 26 May 2022 11:27:23 +1000 Subject: [PATCH 4/6] correct item list number in docs --- docs/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/overview.md b/docs/overview.md index 1342fcb..acc1ff1 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -21,4 +21,4 @@ 8. [Composite Alarms](composite_alarms.md) 9. [Alarms for Custom Metrics](custom_metrics.md) 10. [Dimension Variables](variables.md) -10. [Alarm Tags](alarm_tags.md) \ No newline at end of file +11. [Alarm Tags](alarm_tags.md) \ No newline at end of file From 7f3e088256e42ec5e490ac638ce07e27616a80e4 Mon Sep 17 00:00:00 2001 From: Guslington Date: Thu, 26 May 2022 11:27:57 +1000 Subject: [PATCH 5/6] fix bug where global tags were always being removed and added --- lib/cfnguardian/tagger.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/cfnguardian/tagger.rb b/lib/cfnguardian/tagger.rb index 433c121..17de53e 100644 --- a/lib/cfnguardian/tagger.rb +++ b/lib/cfnguardian/tagger.rb @@ -15,7 +15,6 @@ def tag_alarm(alarm, global_tags={}) new_tags = get_tags(alarm, global_tags) current_tags = get_alarm_tags(alarm_arn) - tags_to_delete = get_tags_to_delete(current_tags, new_tags) if tags_to_delete.any? @@ -27,7 +26,6 @@ def tag_alarm(alarm, global_tags={}) end if tags_changed?(current_tags, new_tags) - # Aws::CloudWatch::Errors::Throttling logger.debug "Updating tags on alarm #{alarm_arn}" @client.tag_resource({ resource_arn: alarm_arn, @@ -38,11 +36,11 @@ def tag_alarm(alarm, global_tags={}) def get_tags(alarm, global_tags) defaults = { - 'guardian:resource:id': alarm.resource_id, - 'guardian:resource:group': alarm.group, - 'guardian:alarm:name': alarm.name, - 'guardian:alarm:metric': alarm.metric_name, - 'guardian:alarm:severity': alarm.alarm_action + 'guardian:resource:id' => alarm.resource_id, + 'guardian:resource:group' => alarm.group, + 'guardian:alarm:name' => alarm.name, + 'guardian:alarm:metric' => alarm.metric_name, + 'guardian:alarm:severity' => alarm.alarm_action } tags = global_tags.merge(defaults) return alarm.tags.merge(tags) @@ -56,7 +54,7 @@ def get_alarm_tags(alarm_arn) end def get_tags_to_delete(current_tags, new_tags) - return current_tags.select {|tag| !new_tags.has_key?(tag.key.to_sym)}.map {|tag| tag.key} + return current_tags.select {|tag| !new_tags.has_key?(tag.key)}.map {|tag| tag.key} end def tags_changed?(current_tags, new_tags) @@ -64,7 +62,7 @@ def tags_changed?(current_tags, new_tags) end def tags_to_hash(tags) - return tags.map {|tag| {tag.key.to_sym => tag.value} }.reduce(Hash.new, :merge) + return tags.map {|tag| {tag.key => tag.value} }.reduce(Hash.new, :merge) end end From fb428a8223a6ce9a4c0ca7b7fb4d6f0621c6b175 Mon Sep 17 00:00:00 2001 From: Guslington Date: Thu, 26 May 2022 11:29:13 +1000 Subject: [PATCH 6/6] increase tagger aws client max attempts to handle api throtteling when tagging lots of alarms --- lib/cfnguardian/tagger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cfnguardian/tagger.rb b/lib/cfnguardian/tagger.rb index 17de53e..164410f 100644 --- a/lib/cfnguardian/tagger.rb +++ b/lib/cfnguardian/tagger.rb @@ -7,7 +7,7 @@ class Tagger include Logging def initialize() - @client = Aws::CloudWatch::Client.new() + @client = Aws::CloudWatch::Client.new(max_attempts: 5) end def tag_alarm(alarm, global_tags={})