From d5b0543cb18e6d2ab99e6e7d2b070c01d39598d1 Mon Sep 17 00:00:00 2001 From: Gerrit 'Nold' Pannek Date: Wed, 28 Jan 2015 13:01:36 +0000 Subject: [PATCH 1/5] Added: Prune reports that match a specific status --- app/models/node.rb | 4 ++-- lib/tasks/prune_reports.rake | 43 +++++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/app/models/node.rb b/app/models/node.rb index d26d456c1..bbf81dda8 100644 --- a/app/models/node.rb +++ b/app/models/node.rb @@ -221,9 +221,9 @@ def find_and_assign_last_inspect_report end end - def prune_reports(cutoff) + def prune_reports(cutoff, condition, status) transaction do - old_report_ids = self.reports.where('reports.time < ?', cutoff).pluck(:id) + old_report_ids = self.reports.where("reports.time < \"#{cutoff}\" and reports.status #{condition} \"#{status}\"").pluck(:id) deleted_count = Report.bulk_delete(old_report_ids) self.find_and_assign_last_inspect_report self.find_and_assign_last_apply_report diff --git a/lib/tasks/prune_reports.rake b/lib/tasks/prune_reports.rake index b12a0c030..5f09e1516 100644 --- a/lib/tasks/prune_reports.rake +++ b/lib/tasks/prune_reports.rake @@ -13,13 +13,34 @@ namespace :reports do } known_units = units.keys.join(',') + statuses = [ + 'changed', + 'unchanged', + 'pending', + 'failed' + ] + known_statuses = statuses.join(',') + + conditions = { + 'is' => '=', + 'not' => '!=' + } + known_conditions = conditions.keys.join(',') + usage = %{ EXAMPLE: # Prune records upto 1 month old: rake reports:prune upto=1 unit=mon + # Prune records upto 1 month old and with status 'unchanged' + rake reports:prune upto=1 unit=mon condition=is status=unchanged + UNITS: Valid units of time are: #{known_units} + +STATUS & CONDITION: + Valid status values are: #{known_statuses} + Valid condition values are: #{known_conditions} }.strip unless ENV['upto'] || ENV['unit'] @@ -43,6 +64,18 @@ UNITS: errors << "You must specify the unit of time, .e.g.: unit={#{known_units}}" \ end + if ( ENV['status'] && ! ENV['condition'] ) || ( ENV['condition'] && ! ENV['status'] ) + errors << "You must specify status AND condition!"\ + end + + if status = ENV['status'] + unless statuses.include?(status) + errors << "I don't know that status. Valid statuses are: #{known_statuses}"\ + end + else + status = "" + end + if errors.present? $stderr.puts errors.map { |error| "ERROR: #{error}" } $stderr.puts @@ -50,15 +83,19 @@ UNITS: exit 1 end + if condition != "!=" + condition = conditions[condition] + end + # Thin query for nodes with reports that may be pruned. # By selecting the 'id' column only, it does not eager load all of the # nodes and definitely not all of the reports, making this much faster. cutoff = Time.now.gmtime - (upto * units[unit].to_i) affected_nodes = Node.select('DISTINCT nodes.id') \ .joins('LEFT OUTER JOIN reports ON reports.node_id = nodes.id') \ - .where(['reports.time < ?', cutoff]) + .where("reports.time < \"#{cutoff}\" and reports.status #{condition} \"#{status}\"") deletion_count = affected_nodes.count - puts "#{Time.now.to_s(:db)}: Deleting reports before #{cutoff} for #{deletion_count} nodes" + puts "#{Time.now.to_s(:db)}: Deleting reports before #{cutoff} and with report status #{condition} \"#{status}\" for #{deletion_count} nodes" pbar = ProgressBar.new('Deleting', deletion_count, STDOUT) deleted_count = 0 @@ -66,7 +103,7 @@ UNITS: begin affected_nodes.each do |node| node.reload # Become a complete object (the query above returns shalow objects with 'id' only) - deleted_count += node.prune_reports(cutoff) + deleted_count += node.prune_reports(cutoff, condition, status) pbar.inc end rescue SignalException From aa0f20d518a97177f80d6e769245e02b35eee5da Mon Sep 17 00:00:00 2001 From: Gerrit 'Nold' Pannek Date: Sun, 1 Feb 2015 12:45:40 +0000 Subject: [PATCH 2/5] Fix: condition parsing --- lib/tasks/prune_reports.rake | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/tasks/prune_reports.rake b/lib/tasks/prune_reports.rake index 5f09e1516..b8651075e 100644 --- a/lib/tasks/prune_reports.rake +++ b/lib/tasks/prune_reports.rake @@ -76,6 +76,14 @@ STATUS & CONDITION: status = "" end + if condition = ENV['condition'] + unless conditions.has_key?(condition) + errors << "I don't know that condition. Valid conditionss are: #{known_conditions}"\ + end + else + condition = "!=" + end + if errors.present? $stderr.puts errors.map { |error| "ERROR: #{error}" } $stderr.puts From 136f5feaa09a5d5b0f1ad9f8037c44eeae63e9bf Mon Sep 17 00:00:00 2001 From: Gerrit 'Nold' Pannek Date: Sun, 1 Feb 2015 15:28:51 +0000 Subject: [PATCH 3/5] Change: Code cleanup --- lib/tasks/prune_reports.rake | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/tasks/prune_reports.rake b/lib/tasks/prune_reports.rake index b8651075e..b4e5bc9cf 100644 --- a/lib/tasks/prune_reports.rake +++ b/lib/tasks/prune_reports.rake @@ -81,7 +81,7 @@ STATUS & CONDITION: errors << "I don't know that condition. Valid conditionss are: #{known_conditions}"\ end else - condition = "!=" + condition = "not" end if errors.present? @@ -91,9 +91,7 @@ STATUS & CONDITION: exit 1 end - if condition != "!=" - condition = conditions[condition] - end + condition = conditions[condition] # Thin query for nodes with reports that may be pruned. # By selecting the 'id' column only, it does not eager load all of the @@ -101,7 +99,7 @@ STATUS & CONDITION: cutoff = Time.now.gmtime - (upto * units[unit].to_i) affected_nodes = Node.select('DISTINCT nodes.id') \ .joins('LEFT OUTER JOIN reports ON reports.node_id = nodes.id') \ - .where("reports.time < \"#{cutoff}\" and reports.status #{condition} \"#{status}\"") + .where("reports.time < ? AND reports.status #{condition} ?", cutoff, status) deletion_count = affected_nodes.count puts "#{Time.now.to_s(:db)}: Deleting reports before #{cutoff} and with report status #{condition} \"#{status}\" for #{deletion_count} nodes" From 7ad084dbe35049fc7dc019f271cc4e51c20c4621 Mon Sep 17 00:00:00 2001 From: Gerrit 'Nold' Pannek Date: Sun, 1 Feb 2015 16:21:34 +0000 Subject: [PATCH 4/5] Fix: Code cleanup in prune_reports --- app/models/node.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/node.rb b/app/models/node.rb index bbf81dda8..8d7c09d1a 100644 --- a/app/models/node.rb +++ b/app/models/node.rb @@ -223,7 +223,7 @@ def find_and_assign_last_inspect_report def prune_reports(cutoff, condition, status) transaction do - old_report_ids = self.reports.where("reports.time < \"#{cutoff}\" and reports.status #{condition} \"#{status}\"").pluck(:id) + old_report_ids = self.reports.where("reports.time < ? AND reports.status #{condition} ?", cutoff, status).pluck(:id) deleted_count = Report.bulk_delete(old_report_ids) self.find_and_assign_last_inspect_report self.find_and_assign_last_apply_report From 8567db6ed2412faa143831cfc6213eb8c2f9829a Mon Sep 17 00:00:00 2001 From: Gerrit 'Nold' Pannek Date: Mon, 2 Feb 2015 06:17:13 +0000 Subject: [PATCH 5/5] Cleanup: removed trailing '\' and extra 's' --- lib/tasks/prune_reports.rake | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/tasks/prune_reports.rake b/lib/tasks/prune_reports.rake index b4e5bc9cf..5227b4a8c 100644 --- a/lib/tasks/prune_reports.rake +++ b/lib/tasks/prune_reports.rake @@ -53,24 +53,24 @@ STATUS & CONDITION: if ENV['upto'] =~ /^\d+$/ upto = ENV['upto'].to_i else - errors << "You must specify how far up you want to prune as an integer, e.g.: upto={some integer}" \ + errors << "You must specify how far up you want to prune as an integer, e.g.: upto={some integer}" end if unit = ENV['unit'] unless units.has_key?(unit) - errors << "I don't know that unit. Valid units are: #{known_units}" \ + errors << "I don't know that unit. Valid units are: #{known_units}" end else - errors << "You must specify the unit of time, .e.g.: unit={#{known_units}}" \ + errors << "You must specify the unit of time, .e.g.: unit={#{known_units}}" end if ( ENV['status'] && ! ENV['condition'] ) || ( ENV['condition'] && ! ENV['status'] ) - errors << "You must specify status AND condition!"\ + errors << "You must specify status AND condition!" end if status = ENV['status'] unless statuses.include?(status) - errors << "I don't know that status. Valid statuses are: #{known_statuses}"\ + errors << "I don't know that status. Valid statuses are: #{known_statuses}" end else status = "" @@ -78,7 +78,7 @@ STATUS & CONDITION: if condition = ENV['condition'] unless conditions.has_key?(condition) - errors << "I don't know that condition. Valid conditionss are: #{known_conditions}"\ + errors << "I don't know that condition. Valid conditions are: #{known_conditions}" end else condition = "not"