[PR] Fixes #35959 - Add structured APT content mode

Co-authored-by: Bernhard Suttner <[email protected]>

from Katello#10420

app/lib/actions/katello/product/content_create.rb

@@ -14,7 +14,7 @@ def plan(root)
                                            arches:        root.format_arches,
                                            label:         root.custom_content_label,
                                            os_versions:   root.os_versions&.join(','),
-                                           content_url:   root.custom_content_path)
+                                           content_url:   root.format_custom_content_path)
               content_id = content_create.output[:response][:id]
               plan_action(Candlepin::Product::ContentAdd, owner: root.product.organization.label,
                                     product_id: root.product.cp_id,
@@ -32,7 +32,7 @@ def plan(root)
                           type:        root.content_type,
                           arches:      root.format_arches,
                           label:       root.custom_content_label,
-                          content_url: root.custom_content_path,
+                          content_url: root.format_custom_content_path,
                           gpg_key_url: root.library_instance.yum_gpg_key_url)
             end
 
@@ -48,7 +48,7 @@ def finalize
             new_content.name = root.name
             new_content.content_type = root.content_type
             new_content.label = root.custom_content_label
-            new_content.content_url = root.custom_content_path
+            new_content.content_url = root.format_custom_content_path
             new_content.vendor = ::Katello::Provider::CUSTOM
           end
 

app/lib/actions/katello/repository/update.rb

@@ -26,7 +26,7 @@ def plan(root, repo_params)
                         :owner => repository.organization.label,
                         :content_id => root.content_id,
                         :name => root.name,
-                        :content_url => root.custom_content_path,
+                        :content_url => root.format_custom_content_path,
                         :gpg_key_url => repository.yum_gpg_key_url,
                         :label => content.label,
                         :type => root.content_type,
@@ -36,7 +36,7 @@ def plan(root, repo_params)
                       )
 
             content.update!(name: root.name,
-                                       content_url: root.custom_content_path,
+                                       content_url: root.format_custom_content_path,
                                        content_type: repository.content_type,
                                        label: content.label,
                                        gpg_url: repository.yum_gpg_key_url)

app/models/katello/root_repository.rb

@@ -447,6 +447,24 @@ def format_arches
       end
     end
 
+    def format_custom_content_path
+      path = custom_content_path
+      if content_type == ::Katello::Repository::DEB_TYPE && Setting['deb_use_structured_content']
+        if deb_components.present? && deb_releases.present?
+          params = []
+          params << "comp=#{deb_components.gsub(" ", ",")}"
+          params << "rel=#{deb_releases.gsub(" ", ",")}"
+          path += "?#{params.join('&')}"
+        else
+          Rails.logger.warn("deb_use_structured_content is set, but repository #{self.name} is lacking deb_releases or deb_components, so we are defaulting to using simple content.")
+          unless Setting['deb_use_simple_publish']
+            Rails.logger.warn("Defaulting to using simple content for repository #{self.name}, but deb_use_simple_publish is not set! This can lead to errors on the clients!")
+          end
+        end
+      end
+      path
+    end
+
     apipie :class, desc: 'A class representing Repository object' do
       name 'Repository'
       refs 'Repository'

app/services/katello/pulp3/api/apt.rb

@@ -23,6 +23,10 @@ def self.add_remove_content_class
         def copy_api
           PulpDebClient::CopyApi.new(api_client)
         end
+
+        def content_release_components_api
+          PulpDebClient::ContentReleaseComponentsApi.new(api_client)
+        end
       end
     end
   end

app/services/katello/pulp3/repository/apt.rb

@@ -31,22 +31,22 @@ def sync_url_params(sync_options)
         end
 
         def mirror_remote_options
-          super.merge(
-            {
-              distributions: repo.deb_releases + "#{' default' unless repo.deb_releases.include? 'default'}"
-            }
-          )
+          if Setting['deb_use_structured_proxy_sync']
+            distributions = "#{repo.deb_releases}"
+          elsif Setting['deb_use_simple_publish']
+            distributions = 'default'
+          else
+            fail("It cannot work to set both deb_use_structured_proxy_sync and deb_use_simple_publish to False! Please set at least one of them to True!")
+          end
+
+          super.merge({distributions: distributions})
         end
 
         def publication_options(repository_version)
           ss = api.signing_services_api.list(name: SIGNING_SERVICE_NAME).results
           popts = super(repository_version)
-          popts.merge!(
-            {
-              structured: true, # publish real suites (e.g. 'stable')
-              simple: true # publish all into 'default'-suite
-            }
-          )
+          popts.merge!({ simple: true }) if Setting['deb_use_simple_publish']
+          popts.merge!({ structured: true })
           popts[:signing_service] = ss[0].pulp_href if ss && ss.length == 1
           popts
         end
@@ -56,7 +56,6 @@ def mirror_publication_options
             # Since we are synchronizing the "default" distribution from the simple publisher on the server,
             # it will be included in the structured publish. Therefore, we MUST NOT use the simple publisher
             # on the proxy, since this would collide!
-            #simple: true,
             structured: true # publish real suites (e.g. 'stable')
           }
         end

lib/katello/plugin.rb

@@ -668,6 +668,24 @@ def katello_template_setting_values(name)
         default: true,
         full_name: N_('Distribute archived content view versions'),
         description: N_("If this is enabled, repositories of content view versions without environments (\"archived\") will be distributed at '/pulp/content/<organization>/content_views/<content view>/X.Y/...'.")
+
+      setting 'deb_use_simple_publish',
+        type: :boolean,
+        default: true,
+        full_name: N_('Use simple publish for deb'),
+        description: N_("If enabled APT repos will publish a catch all Release file under /dists/default/ with a single component named 'all' in it.")
+
+      setting 'deb_use_structured_content',
+        type: :boolean,
+        default: false,
+        full_name: N_('Use structured content for deb clients'),
+        description: N_("If enabled, repo URL's for deb content hosts will be appended with structure information where available.")
+
+      setting 'deb_use_structured_proxy_sync',
+        type: :boolean,
+        default: false,
+        full_name: N_('Sync structured content for proxy syncs'),
+        description: N_('If enabled, any upstream distributions synced to the server will also be used for proxy syncs.')
     end
   end
 

lib/katello/tasks/enable_structured_content_for_deb.rake

@@ -0,0 +1,38 @@
+namespace :katello do
+  desc "Enable or disable the use of structured content for APT clients."
+  task :enable_structured_content_for_deb, ['deb_use_structured_content'] => ['environment', 'dynflow:client', 'check_ping'] do |_t, args|
+    unless ['true', 'false'].include?(args[:deb_use_structured_content])
+      puts 'You must specify if structured content should be enabled or disabled, your options are:'
+      puts '  foreman-rake katello:enable_structured_content_for_deb[true]'
+      puts '  foreman-rake katello:enable_structured_content_for_deb[false]'
+      puts "Note that use of structured content is currently set to '#{Setting['deb_use_structured_content']}'!"
+      puts 'Note that after enabling structured content, you may need to resync your proxies!'
+      exit 1
+    end
+
+    User.current = User.anonymous_api_admin
+    deb_use_structured_content = ActiveModel::Type::Boolean.new.cast(args[:deb_use_structured_content])
+
+    # Force deb_use_simple_publish to true, since we are not yet ready to drop simple publishing!
+    Setting['deb_use_simple_publish'] = true
+
+    # Update deb_use_structured_content and deb_sue_structured_proxy_sync to the value requested by the user:
+    Setting['deb_use_structured_content'] = deb_use_structured_content
+    Setting['deb_use_structured_proxy_sync'] = deb_use_structured_content
+
+    # Ignore repositories where url is not set, since those are presumably empty or used for uploads!
+    roots = Katello::RootRepository.deb_type.where.not(url: nil)
+    roots.each do |root|
+      # Note that we are assuming root.deb_releases must necessarily be set already if url is also set.
+      components = root.deb_components
+      if components.blank? && deb_use_structured_content
+        repo = root.library_instance
+        repo_backend_service = repo.backend_service(SmartProxy.pulp_primary)
+        pulp_components = repo_backend_service.api.content_release_components_api.list(repository_version: repo.version_href)
+        components = pulp_components.results.map(&:component).join(' ')
+      end
+      # We call a repository update action on every deb type root repository with an upstream url:
+      ForemanTasks.sync_task(::Actions::Katello::Repository::Update, root, deb_components: components)
+    end
+  end
+end