diff --git a/manifests/init.pp b/manifests/init.pp index 6c624297..4ac7342c 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -132,6 +132,11 @@ # available, likely results in performance degradation due to I/O blocking and is not recommended in most cases. Modifying this parameter should # be done incrementally with benchmarking at each step to determine an optimal value for your deployment. # +# @param use_legacy_tasking_system +# Use the legacy tasking system with RQ workers instead of the newer tasking system. Do not modify this setting unless you understand the implications for +# performance and stability. Note that any benchmarking you did to find an optimal value of the worker_count parameter will no longer be applicable if you +# change the configured tasking system. The newer tasking system is only supported for Pulpcore releases >= 3.13. +# # @param service_enable # Enable/disable Pulp services at boot. # @@ -197,6 +202,7 @@ Pulpcore::ChecksumTypes $allowed_content_checksums = ['sha224', 'sha256', 'sha384', 'sha512'], String[1] $remote_user_environ_name = 'HTTP_REMOTE_USER', Integer[0] $worker_count = min(8, $facts['processors']['count']), + Boolean $use_legacy_tasking_system = true, Boolean $service_enable = true, Boolean $service_ensure = true, Integer[0] $content_service_worker_count = (2*min(8, $facts['processors']['count']) + 1), diff --git a/spec/acceptance/disable_new_tasking_system_spec.rb b/spec/acceptance/disable_new_tasking_system_spec.rb new file mode 100644 index 00000000..64176c56 --- /dev/null +++ b/spec/acceptance/disable_new_tasking_system_spec.rb @@ -0,0 +1,139 @@ +require 'spec_helper_acceptance' + +describe 'initial configuration with new tasking system' do + certdir = '/etc/pulpcore-certs' + + it_behaves_like 'an idempotent resource' do + let(:manifest) do + <<-PUPPET + class { 'pulpcore': + worker_count => 1, + use_legacy_tasking_system => false, + } + PUPPET + end + end + + describe service('httpd') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-api') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-content') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-resource-manager') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-worker@1') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe port(80) do + it { is_expected.to be_listening } + end + + describe port(443) do + it { is_expected.to be_listening } + end + + describe curl_command("https://#{host_inventory['fqdn']}/pulp/api/v3/status/", cacert: "#{certdir}/ca-cert.pem") do + its(:response_code) { is_expected.to eq(200) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = True/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^0 workers, /) } + its(:stdout) { is_expected.not_to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("ps -e -o command") do + its(:stdout) { is_expected.to match(%r(^/usr/libexec/pulpcore/pulpcore-worker$)) } + its(:stdout) { is_expected.to match(%r(^/usr/libexec/pulpcore/pulpcore-worker --resource-manager$)) } + its(:exit_status) { is_expected.to eq 0 } + end +end + +describe 'disable new tasking system' do + it_behaves_like 'an idempotent resource' do + let(:manifest) do + <<-PUPPET + class { 'pulpcore': + worker_count => 1, + use_legacy_tasking_system => true, + } + PUPPET + end + end + + describe service('httpd') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-api') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-content') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-resource-manager') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-worker@1') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe port(80) do + it { is_expected.to be_listening } + end + + describe port(443) do + it { is_expected.to be_listening } + end + + describe curl_command("https://#{host_inventory['fqdn']}/pulp/api/v3/status/", cacert: "#{certdir}/ca-cert.pem") do + its(:response_code) { is_expected.to eq(200) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = False/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^2 workers, /) } + its(:stdout) { is_expected.to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("ps -e -o command") do + its(:stdout) { is_expected.not_to match(%r(^/usr/libexec/pulpcore/pulpcore-worker$)) } + its(:stdout) { is_expected.not_to match(%r(^/usr/libexec/pulpcore/pulpcore-worker --resource-manager$)) } + its(:exit_status) { is_expected.to eq 0 } + end +end diff --git a/spec/acceptance/enable_new_tasking_system_spec.rb b/spec/acceptance/enable_new_tasking_system_spec.rb new file mode 100644 index 00000000..76afa4e0 --- /dev/null +++ b/spec/acceptance/enable_new_tasking_system_spec.rb @@ -0,0 +1,139 @@ +require 'spec_helper_acceptance' + +describe 'initial configuration with legacy tasking system' do + certdir = '/etc/pulpcore-certs' + + it_behaves_like 'an idempotent resource' do + let(:manifest) do + <<-PUPPET + class { 'pulpcore': + worker_count => 1, + use_legacy_tasking_system => true, + } + PUPPET + end + end + + describe service('httpd') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-api') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-content') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-resource-manager') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-worker@1') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe port(80) do + it { is_expected.to be_listening } + end + + describe port(443) do + it { is_expected.to be_listening } + end + + describe curl_command("https://#{host_inventory['fqdn']}/pulp/api/v3/status/", cacert: "#{certdir}/ca-cert.pem") do + its(:response_code) { is_expected.to eq(200) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = False/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^2 workers, /) } + its(:stdout) { is_expected.to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("ps -e -o command") do + its(:stdout) { is_expected.not_to match(%r(^/usr/libexec/pulpcore/pulpcore-worker$)) } + its(:stdout) { is_expected.not_to match(%r(^/usr/libexec/pulpcore/pulpcore-worker --resource-manager$)) } + its(:exit_status) { is_expected.to eq 0 } + end +end + +describe 'enable new tasking system' do + it_behaves_like 'an idempotent resource' do + let(:manifest) do + <<-PUPPET + class { 'pulpcore': + worker_count => 1, + use_legacy_tasking_system => false, + } + PUPPET + end + end + + describe service('httpd') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-api') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-content') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-resource-manager') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe service('pulpcore-worker@1') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe port(80) do + it { is_expected.to be_listening } + end + + describe port(443) do + it { is_expected.to be_listening } + end + + describe curl_command("https://#{host_inventory['fqdn']}/pulp/api/v3/status/", cacert: "#{certdir}/ca-cert.pem") do + its(:response_code) { is_expected.to eq(200) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("PULP_SETTINGS=/etc/pulp/settings.py pulpcore-manager diffsettings") do + its(:stdout) { is_expected.to match(/^USE_NEW_WORKER_TYPE = True/) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("DJANGO_SETTINGS_MODULE=pulpcore.app.settings PULP_SETTINGS=/etc/pulp/settings.py rq info -c pulpcore.rqconfig") do + its(:stdout) { is_expected.to match(/^0 workers, /) } + its(:stdout) { is_expected.not_to match(/^resource-manager /) } + its(:exit_status) { is_expected.to eq 0 } + end + + describe command("ps -e -o command") do + its(:stdout) { is_expected.to match(%r(^/usr/libexec/pulpcore/pulpcore-worker$)) } + its(:stdout) { is_expected.to match(%r(^/usr/libexec/pulpcore/pulpcore-worker --resource-manager$)) } + its(:exit_status) { is_expected.to eq 0 } + end +end diff --git a/templates/pulpcore-resource-manager.service.erb b/templates/pulpcore-resource-manager.service.erb index cee66dd0..670a71db 100644 --- a/templates/pulpcore-resource-manager.service.erb +++ b/templates/pulpcore-resource-manager.service.erb @@ -11,10 +11,14 @@ User=<%= scope['pulpcore::user'] %> Group=<%= scope['pulpcore::group'] %> WorkingDirectory=<%= scope['pulpcore::user_home'] %> RuntimeDirectory=pulpcore-resource-manager +<% if scope['pulpcore::use_legacy_tasking_system'] %> ExecStart=/usr/libexec/pulpcore/rq worker \ -w pulpcore.tasking.worker.PulpWorker -n resource-manager \ -c 'pulpcore.rqconfig' \ --disable-job-desc-logging +<% else %> +ExecStart=/usr/libexec/pulpcore/pulpcore-worker --resource-manager +<% end %> SyslogIdentifier=pulpcore-resource-manager diff --git a/templates/pulpcore-worker@.service.erb b/templates/pulpcore-worker@.service.erb index f0c5d87d..24dbee3b 100644 --- a/templates/pulpcore-worker@.service.erb +++ b/templates/pulpcore-worker@.service.erb @@ -1,5 +1,9 @@ [Unit] +<% if scope['pulpcore::use_legacy_tasking_system'] %> Description=Pulp RQ Worker +<% else %> +Description=Pulp Worker +<% end %> After=network-online.target Wants=network-online.target @@ -13,10 +17,14 @@ User=<%= scope['pulpcore::user'] %> Group=<%= scope['pulpcore::group'] %> WorkingDirectory=<%= scope['pulpcore::user_home'] %> RuntimeDirectory=pulpcore-worker-%i +<% if scope['pulpcore::use_legacy_tasking_system'] %> ExecStart=/usr/libexec/pulpcore/rq worker \ -w pulpcore.tasking.worker.PulpWorker \ -c 'pulpcore.rqconfig' \ --disable-job-desc-logging +<% else %> +ExecStart=/usr/libexec/pulpcore/pulpcore-worker +<% end %> SyslogIdentifier=pulpcore-worker-%i diff --git a/templates/settings.py.erb b/templates/settings.py.erb index 54e847fb..b943f5bc 100644 --- a/templates/settings.py.erb +++ b/templates/settings.py.erb @@ -40,6 +40,15 @@ REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES = ( ALLOWED_IMPORT_PATHS = <%= scope['pulpcore::allowed_import_path'] %> ALLOWED_EXPORT_PATHS = <%= scope['pulpcore::allowed_export_path'] %> ALLOWED_CONTENT_CHECKSUMS = <%= scope['pulpcore::allowed_content_checksums'] %> +<% if scope['pulpcore::use_legacy_tasking_system'] %> + +# This setting is necessary but not sufficient to use the new tasking system; +# the systemd unit files must also be properly configured for either worker type. +# As such you should NEVER directly modify this setting. Refer to the documentation +# (from your software vendor, if any, otherwise to the community documentation) for +# the correct procedure to change between the new and legacy worker types. +USER_NEW_WORKER_TYPE=True +<% end %> # Derive HTTP/HTTPS via the X-Forwarded-Proto header set by Apache SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')