From 36e474e40502b3de67fb41b9031903aba353eda7 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Wed, 29 Jul 2015 10:55:08 -0700 Subject: [PATCH 01/12] [enhancement] Automatic restore in solo mode When running in solo mode, check remote bucket for existing backup files. If found, fetch and restore using latest backup files, otherwise provision as fresh chef server as normal. --- attributes/default.rb | 3 +- recipes/solo.rb | 82 +++++++++++++++------ templates/default/chef-server-backup.rb.erb | 10 ++- 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/attributes/default.rb b/attributes/default.rb index 6990acb..d28e1db 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -29,6 +29,7 @@ default[:chef_server_populator][:backup][:filename] = 'chef-server-full' default[:chef_server_populator][:backup][:remote][:connection] = {} default[:chef_server_populator][:backup][:remote][:directory] = nil +default[:chef_server_populator][:backup][:remote][:file_prefix] = nil default[:chef_server_populator][:backup][:schedule] = { :minute => '33', :hour => '3' @@ -54,4 +55,4 @@ default[:chef_server_populator][:server_org] = 'inception_llc' #If this is set to nil, the configurator recipe will set it to the server_org. -default[:chef_server_populator][:default_org] = nil +default[:chef_server_populator][:default_org] = nil diff --git a/recipes/solo.rb b/recipes/solo.rb index 7156570..9d6be27 100644 --- a/recipes/solo.rb +++ b/recipes/solo.rb @@ -4,34 +4,70 @@ node.default[:chef_server_populator][:default_org] = node[:chef_server_populator][:server_org] end -include_recipe 'chef-server-populator::org' +# if backup pull files include restore -knife_cmd = "#{node[:chef_server_populator][:knife_exec]}" -knife_opts = "-s https://127.0.0.1/organizations/#{node[:chef_server_populator][:server_org]} -c /etc/opscode/pivotal.rb" - -node[:chef_server_populator][:clients].each do |client, pub_key| - execute "create client: #{client}" do - command "#{knife_cmd} client create #{client} --admin -d #{knife_opts} > /dev/null 2>&1" - not_if "#{knife_cmd} client list #{knife_opts}| tr -d ' ' | grep '^#{client}$'" - retries 5 - end - if(pub_key && node[:chef_server_populator][:base_path]) - pub_key_path = File.join(node[:chef_server_populator][:base_path], pub_key) - execute "remove default public key for #{client}" do - command "chef-server-ctl delete-client-key #{node[:chef_server_populator][:server_org]} #{client} default" - only_if "chef-server-ctl list-client-keys #{node[:chef_server_populator][:server_org]} #{client} | grep 'name: default$'" +if(node[:chef_server_populator][:backup][:remote][:connection]) + chef_gem 'miasma' + require 'miasma' + remote_creds = node[:chef_server_populator][:backup][:remote][:connection] + remote_directory = node[:chef_server_populator][:backup][:remote][:directory] + remote = Miasma.api(:provider => remote_creds[:provider].to_s.downcase, :type => 'storage', :credentials => remote_creds[:credentials]) + remote_bucket = remote.buckets.get(remote_directory) + if(remote_bucket && gz_file = remote_bucket.files.get(File.join(node[:chef_server_populator][:backup][:remote][:file_prefix], 'latest.gz'))) + dump_file = remote_bucket.get(remote_bucket.files.get(File.join(node[:chef_server_populator][:backup][:remote][:file_prefix], 'latest.dump'))) + local_gz = '/tmp/latest.gz' + local_dump = '/tmp/latest.dump' + File.open(local_gz, 'wb') do |file| + while(data = gz_file.body.readpartial(2048)) + file.print data + end end - execute "set public key for: #{client}" do - command "chef-server-ctl add-client-key #{node[:chef_server_populator][:server_org]} #{client} #{pub_key_path} --key-name populator" - not_if "chef-server-ctl list-client-keys #{node[:chef_server_populator][:server_org]} #{client} | grep 'name: populator$'" + File.open(local_dump, 'wb') do |file| + while(data = dump_file.body.readpartial(2048)) + file.print data + end end + node.set[:chef_server_populator][:restore][:file] = local_dump + node.set[:chef_server_populator][:restore][:data] = local_gz end end -execute 'install chef-server-populator cookbook' do - command "#{knife_cmd} cookbook upload chef-server-populator #{knife_opts} -o #{Chef::Config[:cookbook_path].join(':')} --include-dependencies" - only_if do - node[:chef_server_populator][:cookbook_auto_install] +if(local_gz && local_dump) + + include_recipe 'chef-server-populator::restore' + +else + + include_recipe 'chef-server-populator::org' + + knife_cmd = "#{node[:chef_server_populator][:knife_exec]}" + knife_opts = "-s https://127.0.0.1/organizations/#{node[:chef_server_populator][:server_org]} -c /etc/opscode/pivotal.rb" + + node[:chef_server_populator][:clients].each do |client, pub_key| + execute "create client: #{client}" do + command "#{knife_cmd} client create #{client} --admin -d #{knife_opts} > /dev/null 2>&1" + not_if "#{knife_cmd} client list #{knife_opts}| tr -d ' ' | grep '^#{client}$'" + retries 5 + end + if(pub_key && node[:chef_server_populator][:base_path]) + pub_key_path = File.join(node[:chef_server_populator][:base_path], pub_key) + execute "remove default public key for #{client}" do + command "chef-server-ctl delete-client-key #{node[:chef_server_populator][:server_org]} #{client} default" + only_if "chef-server-ctl list-client-keys #{node[:chef_server_populator][:server_org]} #{client} | grep 'name: default$'" + end + execute "set public key for: #{client}" do + command "chef-server-ctl add-client-key #{node[:chef_server_populator][:server_org]} #{client} #{pub_key_path} --key-name populator" + not_if "chef-server-ctl list-client-keys #{node[:chef_server_populator][:server_org]} #{client} | grep 'name: populator$'" + end + end end - retries 5 + + execute 'install chef-server-populator cookbook' do + command "#{knife_cmd} cookbook upload chef-server-populator #{knife_opts} -o #{Chef::Config[:cookbook_path].join(':')} --include-dependencies" + only_if do + node[:chef_server_populator][:cookbook_auto_install] + end + retries 5 + end + end diff --git a/templates/default/chef-server-backup.rb.erb b/templates/default/chef-server-backup.rb.erb index c01faca..a361a6c 100644 --- a/templates/default/chef-server-backup.rb.erb +++ b/templates/default/chef-server-backup.rb.erb @@ -75,6 +75,10 @@ remote_directory = [:remote, :directory].inject(config) do |memo, key| memo[key] || break end +remote_prefix = [:remote, :prefix].inject(config) do |memo, key| + memo[key] || break +end + if(remote_creds) if(remote_directory) remote = Miasma.api(:provider => remote_creds[:provider].to_s.downcase, :type => 'storage', :credentials => remote_creds) @@ -82,7 +86,11 @@ if(remote_creds) [db_file, data_file].each do |file| [ :date_stamped, :latest ].each do |type| remote_file = Miasma::Models::Storage::File.new(directory) - remote_file.name = type == :date_stamped ? File.basename(file) : "latest#{File.extname(file)}" + remote_file.name = File.join(*[ + remote_prefix, + type == :date_stamped ? File.basename(file) : "latest#{File.extname(file)}" + ].flatten.compact + ) remote_file.body = File.open(file, 'r') remote_file.save end From 4573dad1338cce0f8c0c1f818014d3a858607510 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 30 Jul 2015 07:53:23 -0700 Subject: [PATCH 02/12] Set the omnibus bin path first to prevent using system ruby --- recipes/backups.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/backups.rb b/recipes/backups.rb index d908880..b37f79c 100644 --- a/recipes/backups.rb +++ b/recipes/backups.rb @@ -56,5 +56,5 @@ node[:chef_server_populator][:backup][:schedule].each do |k,v| send(k,v) end - path "$PATH:/opt/chef/embedded/bin/" + path "/opt/chef/embedded/bin/:$PATH" end From bd427e30649daae5f284c1d540b25d998879e4fd Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 30 Jul 2015 07:55:14 -0700 Subject: [PATCH 03/12] Reference the credentials within the creds config on api setup --- templates/default/chef-server-backup.rb.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default/chef-server-backup.rb.erb b/templates/default/chef-server-backup.rb.erb index a361a6c..462f0ea 100644 --- a/templates/default/chef-server-backup.rb.erb +++ b/templates/default/chef-server-backup.rb.erb @@ -81,7 +81,7 @@ end if(remote_creds) if(remote_directory) - remote = Miasma.api(:provider => remote_creds[:provider].to_s.downcase, :type => 'storage', :credentials => remote_creds) + remote = Miasma.api(:provider => remote_creds[:provider].to_s.downcase, :type => 'storage', :credentials => remote_creds[:credentials]) directory = remote.buckets.get(remote_directory) [db_file, data_file].each do |file| [ :date_stamped, :latest ].each do |type| From 4faa3f4ad1d0893fd79afac8e1a10f8bc45dee9f Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 30 Jul 2015 08:56:05 -0700 Subject: [PATCH 04/12] Use correct configuration key for file prefix string --- templates/default/chef-server-backup.rb.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default/chef-server-backup.rb.erb b/templates/default/chef-server-backup.rb.erb index 462f0ea..34e98ef 100644 --- a/templates/default/chef-server-backup.rb.erb +++ b/templates/default/chef-server-backup.rb.erb @@ -75,7 +75,7 @@ remote_directory = [:remote, :directory].inject(config) do |memo, key| memo[key] || break end -remote_prefix = [:remote, :prefix].inject(config) do |memo, key| +remote_prefix = [:remote, :file_prefix].inject(config) do |memo, key| memo[key] || break end From 7995d4a1b85491d4b205c60afdaf14040e8511c0 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 30 Jul 2015 09:04:59 -0700 Subject: [PATCH 05/12] Use build_version value from server manifest --- templates/default/chef-server-backup.rb.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default/chef-server-backup.rb.erb b/templates/default/chef-server-backup.rb.erb index 34e98ef..0e580a1 100644 --- a/templates/default/chef-server-backup.rb.erb +++ b/templates/default/chef-server-backup.rb.erb @@ -25,7 +25,7 @@ server_manifest = MultiJson.load( prefix = [ Time.now.to_i, - "ver_#{server_manifest[:version]}", + "ver_#{server_manifest[:build_version]}", config[:filename] ].join('-') From de7e965fc3ffe2a8dd8f62076397ff4cc7d88ef2 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 30 Jul 2015 13:25:40 -0700 Subject: [PATCH 06/12] Fix file extension when fetching latest from remote store --- recipes/solo.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/solo.rb b/recipes/solo.rb index 9d6be27..a3f8d0d 100644 --- a/recipes/solo.rb +++ b/recipes/solo.rb @@ -13,9 +13,9 @@ remote_directory = node[:chef_server_populator][:backup][:remote][:directory] remote = Miasma.api(:provider => remote_creds[:provider].to_s.downcase, :type => 'storage', :credentials => remote_creds[:credentials]) remote_bucket = remote.buckets.get(remote_directory) - if(remote_bucket && gz_file = remote_bucket.files.get(File.join(node[:chef_server_populator][:backup][:remote][:file_prefix], 'latest.gz'))) + if(remote_bucket && gz_file = remote_bucket.files.get(File.join(node[:chef_server_populator][:backup][:remote][:file_prefix], 'latest.tgz'))) dump_file = remote_bucket.get(remote_bucket.files.get(File.join(node[:chef_server_populator][:backup][:remote][:file_prefix], 'latest.dump'))) - local_gz = '/tmp/latest.gz' + local_gz = '/tmp/latest.tgz' local_dump = '/tmp/latest.dump' File.open(local_gz, 'wb') do |file| while(data = gz_file.body.readpartial(2048)) From e0628e2b32d0cd5ac06d534d44978decc1bd7ad0 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 30 Jul 2015 13:39:54 -0700 Subject: [PATCH 07/12] Remove duplicate get calls the to storage api --- recipes/solo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/solo.rb b/recipes/solo.rb index a3f8d0d..3ab9465 100644 --- a/recipes/solo.rb +++ b/recipes/solo.rb @@ -14,7 +14,7 @@ remote = Miasma.api(:provider => remote_creds[:provider].to_s.downcase, :type => 'storage', :credentials => remote_creds[:credentials]) remote_bucket = remote.buckets.get(remote_directory) if(remote_bucket && gz_file = remote_bucket.files.get(File.join(node[:chef_server_populator][:backup][:remote][:file_prefix], 'latest.tgz'))) - dump_file = remote_bucket.get(remote_bucket.files.get(File.join(node[:chef_server_populator][:backup][:remote][:file_prefix], 'latest.dump'))) + dump_file = remote_bucket.files.get(File.join(node[:chef_server_populator][:backup][:remote][:file_prefix], 'latest.dump')) local_gz = '/tmp/latest.tgz' local_dump = '/tmp/latest.dump' File.open(local_gz, 'wb') do |file| From 1c959b44aac634dfe3f29f69a4caca155458cd2b Mon Sep 17 00:00:00 2001 From: Pete Cheslock Date: Thu, 30 Jul 2015 17:31:34 -0400 Subject: [PATCH 08/12] Temp hack to wait for PG to start before restore --- recipes/restore.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/recipes/restore.rb b/recipes/restore.rb index 76014ef..4f388a0 100644 --- a/recipes/restore.rb +++ b/recipes/restore.rb @@ -24,9 +24,17 @@ creates '/etc/opscode/restore.json' end +ruby_block 'sleep10s' do + block do + sleep 10 + end + action :nothing +end + execute 'restore chef server start postgres' do command 'chef-server-ctl start postgresql' creates '/etc/opscode/restore.json' + notifies :run, 'ruby_block[sleep10s]', :immediately end #Drop and Restore entire chef database from file From 9b192fbd3984ea400d96431bfeeeaca6d24ef9a7 Mon Sep 17 00:00:00 2001 From: Pete Cheslock Date: Thu, 30 Jul 2015 18:02:59 -0400 Subject: [PATCH 09/12] Add moree sleeps to things --- recipes/restore.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/recipes/restore.rb b/recipes/restore.rb index 4f388a0..6872a62 100644 --- a/recipes/restore.rb +++ b/recipes/restore.rb @@ -24,17 +24,14 @@ creates '/etc/opscode/restore.json' end -ruby_block 'sleep10s' do - block do - sleep 10 - end - action :nothing -end - execute 'restore chef server start postgres' do command 'chef-server-ctl start postgresql' creates '/etc/opscode/restore.json' - notifies :run, 'ruby_block[sleep10s]', :immediately +end + +execute 'restore chef server wait for postgresql' do + command 'sleep 10' + creates '/etc/opscode/restore.json' end #Drop and Restore entire chef database from file @@ -44,6 +41,11 @@ creates '/etc/opscode/restore.json' end +execute 'restore chef server wait for postgresql post restore' do + command 'sleep 10' + creates '/etc/opscode/restore.json' +end + execute 'remove existing data' do command "rm -rf /var/opt/opscode /etc/opscode" creates '/etc/opscode/restore.json' From 93e1bec48c1637796e914147e6d4a662d7b8abd2 Mon Sep 17 00:00:00 2001 From: Pete Cheslock Date: Thu, 30 Jul 2015 18:25:01 -0400 Subject: [PATCH 10/12] Make chef wait longer for postgresql --- recipes/restore.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/restore.rb b/recipes/restore.rb index 6872a62..650cf2d 100644 --- a/recipes/restore.rb +++ b/recipes/restore.rb @@ -42,7 +42,7 @@ end execute 'restore chef server wait for postgresql post restore' do - command 'sleep 10' + command 'sleep 30' creates '/etc/opscode/restore.json' end From 2f89ce552a3074b5ed96f81dbc85d7711c2d573f Mon Sep 17 00:00:00 2001 From: Pete Cheslock Date: Thu, 30 Jul 2015 18:40:44 -0400 Subject: [PATCH 11/12] Wait for postgresql to fully stop after restore so it doesn't fail later on restart --- recipes/restore.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/recipes/restore.rb b/recipes/restore.rb index 650cf2d..717eff1 100644 --- a/recipes/restore.rb +++ b/recipes/restore.rb @@ -41,8 +41,13 @@ creates '/etc/opscode/restore.json' end -execute 'restore chef server wait for postgresql post restore' do - command 'sleep 30' +execute 'shutdown postgresql after restore' do + command 'chef-server-ctl stop postgresql' + creates '/etc/opscode/restore.json' +end + +execute 'wait for postgresql to stop' do + command 'sleep 180' creates '/etc/opscode/restore.json' end From b2221741e7c8fe835782096354cb98238e83959c Mon Sep 17 00:00:00 2001 From: Pete Cheslock Date: Thu, 30 Jul 2015 18:51:39 -0400 Subject: [PATCH 12/12] Assuming everything worked 20 seconds should be enough --- recipes/restore.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/restore.rb b/recipes/restore.rb index 717eff1..3ab88af 100644 --- a/recipes/restore.rb +++ b/recipes/restore.rb @@ -47,7 +47,7 @@ end execute 'wait for postgresql to stop' do - command 'sleep 180' + command 'sleep 20' creates '/etc/opscode/restore.json' end