From 68ba6691bce4036117420acc35250c318c029667 Mon Sep 17 00:00:00 2001 From: Quirin Pamp Date: Wed, 21 Jun 2023 18:09:02 +0200 Subject: [PATCH] [PR] Fixes #35959 - Add structured APT content mode Co-authored-by: Bernhard Suttner from https://github.com/Katello/katello/pull/10420 --- .../actions/katello/product/content_create.rb | 6 +-- app/lib/actions/katello/repository/update.rb | 4 +- app/models/katello/root_repository.rb | 18 +++++++++ app/services/katello/pulp3/api/apt.rb | 4 ++ app/services/katello/pulp3/repository/apt.rb | 23 ++++++----- lib/katello/plugin.rb | 18 +++++++++ .../enable_structured_content_for_deb.rake | 38 +++++++++++++++++++ 7 files changed, 94 insertions(+), 17 deletions(-) create mode 100644 lib/katello/tasks/enable_structured_content_for_deb.rake diff --git a/app/lib/actions/katello/product/content_create.rb b/app/lib/actions/katello/product/content_create.rb index db1866ef253..eaf8b592d35 100644 --- a/app/lib/actions/katello/product/content_create.rb +++ b/app/lib/actions/katello/product/content_create.rb @@ -14,7 +14,7 @@ def plan(root) arches: root.format_arches, label: root.custom_content_label, os_versions: root.os_versions&.join(','), - content_url: root.custom_content_path) + content_url: root.format_custom_content_path) content_id = content_create.output[:response][:id] plan_action(Candlepin::Product::ContentAdd, owner: root.product.organization.label, product_id: root.product.cp_id, @@ -32,7 +32,7 @@ def plan(root) type: root.content_type, arches: root.format_arches, label: root.custom_content_label, - content_url: root.custom_content_path, + content_url: root.format_custom_content_path, gpg_key_url: root.library_instance.yum_gpg_key_url) end @@ -48,7 +48,7 @@ def finalize new_content.name = root.name new_content.content_type = root.content_type new_content.label = root.custom_content_label - new_content.content_url = root.custom_content_path + new_content.content_url = root.format_custom_content_path new_content.vendor = ::Katello::Provider::CUSTOM end diff --git a/app/lib/actions/katello/repository/update.rb b/app/lib/actions/katello/repository/update.rb index 476736fbb18..f21263f9da4 100644 --- a/app/lib/actions/katello/repository/update.rb +++ b/app/lib/actions/katello/repository/update.rb @@ -26,7 +26,7 @@ def plan(root, repo_params) :owner => repository.organization.label, :content_id => root.content_id, :name => root.name, - :content_url => root.custom_content_path, + :content_url => root.format_custom_content_path, :gpg_key_url => repository.yum_gpg_key_url, :label => content.label, :type => root.content_type, @@ -36,7 +36,7 @@ def plan(root, repo_params) ) content.update!(name: root.name, - content_url: root.custom_content_path, + content_url: root.format_custom_content_path, content_type: repository.content_type, label: content.label, gpg_url: repository.yum_gpg_key_url) diff --git a/app/models/katello/root_repository.rb b/app/models/katello/root_repository.rb index 6b6689a404e..c70be50bc2f 100644 --- a/app/models/katello/root_repository.rb +++ b/app/models/katello/root_repository.rb @@ -447,6 +447,24 @@ def format_arches end end + def format_custom_content_path + path = custom_content_path + if content_type == ::Katello::Repository::DEB_TYPE && Setting['deb_use_structured_content'] + if deb_components.present? && deb_releases.present? + params = [] + params << "comp=#{deb_components.gsub(" ", ",")}" + params << "rel=#{deb_releases.gsub(" ", ",")}" + path += "?#{params.join('&')}" + else + Rails.logger.warn("deb_use_structured_content is set, but repository #{self.name} is lacking deb_releases or deb_components, so we are defaulting to using simple content.") + unless Setting['deb_use_simple_publish'] + Rails.logger.warn("Defaulting to using simple content for repository #{self.name}, but deb_use_simple_publish is not set! This can lead to errors on the clients!") + end + end + end + path + end + apipie :class, desc: 'A class representing Repository object' do name 'Repository' refs 'Repository' diff --git a/app/services/katello/pulp3/api/apt.rb b/app/services/katello/pulp3/api/apt.rb index 1c09ce06ec7..e8966142fd7 100644 --- a/app/services/katello/pulp3/api/apt.rb +++ b/app/services/katello/pulp3/api/apt.rb @@ -23,6 +23,10 @@ def self.add_remove_content_class def copy_api PulpDebClient::CopyApi.new(api_client) end + + def content_release_components_api + PulpDebClient::ContentReleaseComponentsApi.new(api_client) + end end end end diff --git a/app/services/katello/pulp3/repository/apt.rb b/app/services/katello/pulp3/repository/apt.rb index fd81ae1ea62..e2a97c97f5a 100644 --- a/app/services/katello/pulp3/repository/apt.rb +++ b/app/services/katello/pulp3/repository/apt.rb @@ -31,22 +31,22 @@ def sync_url_params(sync_options) end def mirror_remote_options - super.merge( - { - distributions: repo.deb_releases + "#{' default' unless repo.deb_releases.include? 'default'}" - } - ) + if Setting['deb_use_structured_proxy_sync'] + distributions = "#{repo.deb_releases}" + elsif Setting['deb_use_simple_publish'] + distributions = 'default' + else + fail("It cannot work to set both deb_use_structured_proxy_sync and deb_use_simple_publish to False! Please set at least one of them to True!") + end + + super.merge({distributions: distributions}) end def publication_options(repository_version) ss = api.signing_services_api.list(name: SIGNING_SERVICE_NAME).results popts = super(repository_version) - popts.merge!( - { - structured: true, # publish real suites (e.g. 'stable') - simple: true # publish all into 'default'-suite - } - ) + popts.merge!({ simple: true }) if Setting['deb_use_simple_publish'] + popts.merge!({ structured: true }) popts[:signing_service] = ss[0].pulp_href if ss && ss.length == 1 popts end @@ -56,7 +56,6 @@ def mirror_publication_options # Since we are synchronizing the "default" distribution from the simple publisher on the server, # it will be included in the structured publish. Therefore, we MUST NOT use the simple publisher # on the proxy, since this would collide! - #simple: true, structured: true # publish real suites (e.g. 'stable') } end diff --git a/lib/katello/plugin.rb b/lib/katello/plugin.rb index 2b7e14b5bda..9ffd755cc86 100644 --- a/lib/katello/plugin.rb +++ b/lib/katello/plugin.rb @@ -668,6 +668,24 @@ def katello_template_setting_values(name) default: true, full_name: N_('Distribute archived content view versions'), description: N_("If this is enabled, repositories of content view versions without environments (\"archived\") will be distributed at '/pulp/content//content_views//X.Y/...'.") + + setting 'deb_use_simple_publish', + type: :boolean, + default: true, + full_name: N_('Use simple publish for deb'), + description: N_("If enabled APT repos will publish a catch all Release file under /dists/default/ with a single component named 'all' in it.") + + setting 'deb_use_structured_content', + type: :boolean, + default: false, + full_name: N_('Use structured content for deb clients'), + description: N_("If enabled, repo URL's for deb content hosts will be appended with structure information where available.") + + setting 'deb_use_structured_proxy_sync', + type: :boolean, + default: false, + full_name: N_('Sync structured content for proxy syncs'), + description: N_('If enabled, any upstream distributions synced to the server will also be used for proxy syncs.') end end diff --git a/lib/katello/tasks/enable_structured_content_for_deb.rake b/lib/katello/tasks/enable_structured_content_for_deb.rake new file mode 100644 index 00000000000..6bc11fdb5c3 --- /dev/null +++ b/lib/katello/tasks/enable_structured_content_for_deb.rake @@ -0,0 +1,38 @@ +namespace :katello do + desc "Enable or disable the use of structured content for APT clients." + task :enable_structured_content_for_deb, ['deb_use_structured_content'] => ['environment', 'dynflow:client', 'check_ping'] do |_t, args| + unless ['true', 'false'].include?(args[:deb_use_structured_content]) + puts 'You must specify if structured content should be enabled or disabled, your options are:' + puts ' foreman-rake katello:enable_structured_content_for_deb[true]' + puts ' foreman-rake katello:enable_structured_content_for_deb[false]' + puts "Note that use of structured content is currently set to '#{Setting['deb_use_structured_content']}'!" + puts 'Note that after enabling structured content, you may need to resync your proxies!' + exit 1 + end + + User.current = User.anonymous_api_admin + deb_use_structured_content = ActiveModel::Type::Boolean.new.cast(args[:deb_use_structured_content]) + + # Force deb_use_simple_publish to true, since we are not yet ready to drop simple publishing! + Setting['deb_use_simple_publish'] = true + + # Update deb_use_structured_content and deb_sue_structured_proxy_sync to the value requested by the user: + Setting['deb_use_structured_content'] = deb_use_structured_content + Setting['deb_use_structured_proxy_sync'] = deb_use_structured_content + + # Ignore repositories where url is not set, since those are presumably empty or used for uploads! + roots = Katello::RootRepository.deb_type.where.not(url: nil) + roots.each do |root| + # Note that we are assuming root.deb_releases must necessarily be set already if url is also set. + components = root.deb_components + if components.blank? && deb_use_structured_content + repo = root.library_instance + repo_backend_service = repo.backend_service(SmartProxy.pulp_primary) + pulp_components = repo_backend_service.api.content_release_components_api.list(repository_version: repo.version_href) + components = pulp_components.results.map(&:component).join(' ') + end + # We call a repository update action on every deb type root repository with an upstream url: + ForemanTasks.sync_task(::Actions::Katello::Repository::Update, root, deb_components: components) + end + end +end