diff --git a/manifests/init.pp b/manifests/init.pp index 35d2db0d..4460a9a8 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -132,6 +132,11 @@ # available, likely results in performance degradation due to I/O blocking and is not recommended in most cases. Modifying this parameter should # be done incrementally with benchmarking at each step to determine an optimal value for your deployment. # +# @param use_rq_tasking_system +# Use the older RQ workers tasking system instead of the newer PostgreSQL tasking system introduced in Pulpcore 3.14. +# Any benchmarking you did to optimize worker_count or other tasking related parameters will no longer be accurate after changing the tasking system. +# Do not modify this setting unless you understand the implications for performance and stability. +# # @param service_enable # Enable/disable Pulp services at boot. # @@ -203,6 +208,7 @@ Pulpcore::ChecksumTypes $allowed_content_checksums = ['sha224', 'sha256', 'sha384', 'sha512'], String[1] $remote_user_environ_name = 'HTTP_REMOTE_USER', Integer[0] $worker_count = min(8, $facts['processors']['count']), + Boolean $use_rq_tasking_system = false, Boolean $service_enable = true, Boolean $service_ensure = true, Integer[0] $content_service_worker_count = (2*min(8, $facts['processors']['count']) + 1), diff --git a/manifests/service.pp b/manifests/service.pp index d07aa05d..8861f7ed 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -18,10 +18,18 @@ service_content => template('pulpcore/pulpcore-content.service.erb'), } - systemd::unit_file { 'pulpcore-resource-manager.service': - content => template('pulpcore/pulpcore-resource-manager.service.erb'), - active => $pulpcore::service_ensure, - enable => $pulpcore::service_enable, + if $pulpcore::use_rq_tasking_system { + systemd::unit_file { 'pulpcore-resource-manager.service': + content => template('pulpcore/pulpcore-resource-manager.service.erb'), + active => $pulpcore::service_ensure, + enable => $pulpcore::service_enable, + } + } else { + systemd::unit_file { 'pulpcore-resource-manager.service': + ensure => 'absent', + active => false, + enable => false, + } } systemd::unit_file { 'pulpcore-worker@.service': diff --git a/spec/acceptance/basic_spec.rb b/spec/acceptance/basic_spec.rb index b0e1e03f..c5d89c2e 100644 --- a/spec/acceptance/basic_spec.rb +++ b/spec/acceptance/basic_spec.rb @@ -29,8 +29,8 @@ class { 'pulpcore': end describe service('pulpcore-resource-manager') do - it { is_expected.to be_enabled } - it { is_expected.to be_running } + it { is_expected.not_to be_enabled } + it { is_expected.not_to be_running } end describe service('pulpcore-worker@1') do @@ -73,6 +73,17 @@ class { 'pulpcore': its(:body) { is_expected.to contain('artifacts_list') } its(:exit_status) { is_expected.to eq 0 } end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = True/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^0 workers, /) } + its(:stdout) { is_expected.not_to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end end describe 'reducing worker count' do @@ -102,8 +113,8 @@ class { 'pulpcore': end describe service('pulpcore-resource-manager') do - it { is_expected.to be_enabled } - it { is_expected.to be_running } + it { is_expected.not_to be_enabled } + it { is_expected.not_to be_running } end describe service('pulpcore-worker@1') do diff --git a/spec/acceptance/disable_new_tasking_system_spec.rb b/spec/acceptance/disable_new_tasking_system_spec.rb new file mode 100644 index 00000000..a64c9e7d --- /dev/null +++ b/spec/acceptance/disable_new_tasking_system_spec.rb @@ -0,0 +1,137 @@ +require 'spec_helper_acceptance' + +describe 'change configuration from postgresql tasking system to rq tasking system' do + certdir = '/etc/pulpcore-certs' + + context 'initial configuration with newer postgresql tasking system' do + it_behaves_like 'an idempotent resource' do + let(:manifest) do + <<-PUPPET + class { 'pulpcore': + worker_count => 1, + use_rq_tasking_system => false, + } + PUPPET + end + end + + describe service('httpd') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-api') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-content') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe file('/etc/systemd/system/pulpcore-resource-manager.service') do + it { is_expected.not_to exist } + end + + describe service('pulpcore-resource-manager') do + it { is_expected.not_to be_enabled } + it { is_expected.not_to be_running } + end + + describe service('pulpcore-worker@1') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe port(80) do + it { is_expected.to be_listening } + end + + describe port(443) do + it { is_expected.to be_listening } + end + + describe curl_command("https://#{host_inventory['fqdn']}/pulp/api/v3/status/", cacert: "#{certdir}/ca-cert.pem") do + its(:response_code) { is_expected.to eq(200) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = True/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^0 workers, /) } + its(:stdout) { is_expected.not_to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end + end + + context 'reconfigure pulpcore to use older rq worker tasking system' do + it_behaves_like 'an idempotent resource' do + let(:manifest) do + <<-PUPPET + class { 'pulpcore': + worker_count => 1, + use_rq_tasking_system => true, + } + PUPPET + end + end + + describe service('httpd') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-api') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-content') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe file('/etc/systemd/system/pulpcore-resource-manager.service') do + it { is_expected.to exist } + end + + describe service('pulpcore-resource-manager') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-worker@1') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe port(80) do + it { is_expected.to be_listening } + end + + describe port(443) do + it { is_expected.to be_listening } + end + + describe curl_command("https://#{host_inventory['fqdn']}/pulp/api/v3/status/", cacert: "#{certdir}/ca-cert.pem") do + its(:response_code) { is_expected.to eq(200) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = False/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^2 workers, /) } + its(:stdout) { is_expected.to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end + end +end diff --git a/spec/acceptance/enable_new_tasking_system_spec.rb b/spec/acceptance/enable_new_tasking_system_spec.rb new file mode 100644 index 00000000..058b8380 --- /dev/null +++ b/spec/acceptance/enable_new_tasking_system_spec.rb @@ -0,0 +1,137 @@ +require 'spec_helper_acceptance' + +describe 'change configuration from rq tasking system to postgresql tasking system' do + certdir = '/etc/pulpcore-certs' + + context 'initial configuration with older rq worker tasking system' do + it_behaves_like 'an idempotent resource' do + let(:manifest) do + <<-PUPPET + class { 'pulpcore': + worker_count => 1, + use_rq_tasking_system => true, + } + PUPPET + end + end + + describe service('httpd') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-api') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-content') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe file('/etc/systemd/system/pulpcore-resource-manager.service') do + it { is_expected.to exist } + end + + describe service('pulpcore-resource-manager') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-worker@1') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe port(80) do + it { is_expected.to be_listening } + end + + describe port(443) do + it { is_expected.to be_listening } + end + + describe curl_command("https://#{host_inventory['fqdn']}/pulp/api/v3/status/", cacert: "#{certdir}/ca-cert.pem") do + its(:response_code) { is_expected.to eq(200) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = False/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^2 workers, /) } + its(:stdout) { is_expected.to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end + end + + context 'reconfigure pulpcore to use newer postgresql tasking system' do + it_behaves_like 'an idempotent resource' do + let(:manifest) do + <<-PUPPET + class { 'pulpcore': + worker_count => 1, + use_rq_tasking_system => false, + } + PUPPET + end + end + + describe service('httpd') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-api') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-content') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe file('/etc/systemd/system/pulpcore-resource-manager.service') do + it { is_expected.not_to exist } + end + + describe service('pulpcore-resource-manager') do + it { is_expected.not_to be_enabled } + it { is_expected.not_to be_running } + end + + describe service('pulpcore-worker@1') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe port(80) do + it { is_expected.to be_listening } + end + + describe port(443) do + it { is_expected.to be_listening } + end + + describe curl_command("https://#{host_inventory['fqdn']}/pulp/api/v3/status/", cacert: "#{certdir}/ca-cert.pem") do + its(:response_code) { is_expected.to eq(200) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = True/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^0 workers, /) } + its(:stdout) { is_expected.not_to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end + end +end diff --git a/spec/acceptance/plugins_spec.rb b/spec/acceptance/plugins_spec.rb index a47e7d72..f224244d 100644 --- a/spec/acceptance/plugins_spec.rb +++ b/spec/acceptance/plugins_spec.rb @@ -37,8 +37,8 @@ end describe service('pulpcore-resource-manager') do - it { is_expected.to be_enabled } - it { is_expected.to be_running } + it { is_expected.not_to be_enabled } + it { is_expected.not_to be_running } end describe port(80) do diff --git a/spec/acceptance/use_pulp2_content_route_spec.rb b/spec/acceptance/use_pulp2_content_route_spec.rb index 2017d263..8c52641a 100644 --- a/spec/acceptance/use_pulp2_content_route_spec.rb +++ b/spec/acceptance/use_pulp2_content_route_spec.rb @@ -40,8 +40,8 @@ class { 'pulpcore::plugin::rpm': end describe service('pulpcore-resource-manager') do - it { is_expected.to be_enabled } - it { is_expected.to be_running } + it { is_expected.not_to be_enabled } + it { is_expected.not_to be_running } end describe port(80) do diff --git a/spec/classes/pulpcore_spec.rb b/spec/classes/pulpcore_spec.rb index 8384b36e..c8c74faa 100644 --- a/spec/classes/pulpcore_spec.rb +++ b/spec/classes/pulpcore_spec.rb @@ -128,7 +128,7 @@ is_expected.to contain_systemd__unit_file('pulpcore-content.socket') is_expected.to contain_systemd__unit_file('pulpcore-content.service') is_expected.to contain_file('/etc/systemd/system/pulpcore-content.socket').that_comes_before('Service[pulpcore-content.service]') - is_expected.to contain_systemd__unit_file('pulpcore-resource-manager.service') + is_expected.to contain_systemd__unit_file('pulpcore-resource-manager.service').with_ensure('absent') is_expected.to contain_systemd__unit_file('pulpcore-worker@.service') is_expected.to contain_service("pulpcore-worker@1.service").with_ensure(true) is_expected.not_to contain_service("pulpcore-worker@2.service") diff --git a/templates/settings.py.erb b/templates/settings.py.erb index 06ea173a..7207c1d5 100644 --- a/templates/settings.py.erb +++ b/templates/settings.py.erb @@ -23,6 +23,8 @@ REDIS_HOST = "localhost" REDIS_PORT = "<%= scope['redis::port'] %>" REDIS_DB = <%= scope['pulpcore::redis_db'] %> +USE_NEW_WORKER_TYPE = <%= scope['pulpcore::use_rq_tasking_system'] ? "False" : "True" %> + MEDIA_ROOT = "<%= scope['pulpcore::media_root'] %>" STATIC_ROOT = "<%= scope['pulpcore::static_root'] %>" STATIC_URL = "<%= scope['pulpcore::static_url'] %>"