Fixes #35270 - Enable boot image download for iso images

* Implement fetch and extract boot image
* Implement class for file extraction with isoinfo
* Add capability for archive extraction
* Separate logging and file writing tasks
* Add additional API endpoint /tftp/boot_image/ for boot images

Co-Authored-By: Ewoud Kohl van Wijngaarden <[email protected]>

config/settings.d/tftp.yml.example

@@ -1,5 +1,5 @@
 ---
-# Can be true, false, or http/https to enable just one of the protocols
+# Can be true or false
 :enabled: false
 
 #:tftproot: /var/lib/tftpboot
@@ -13,3 +13,8 @@
 # Defines the default certificate action for certificate checking.
 # When false, the argument --no-check-certificate will be used.
 #:verify_server_cert: true
+
+# Defines the default folder to provide boot images. It becomes important when
+# automating the extraction process as it is done for the Ubuntu Autoinstall
+# procedure.
+#:boot_image_root: /var/lib/foreman-proxy/tftp/boot_images

lib/proxy/archive_extract.rb

@@ -0,0 +1,33 @@
+module Proxy
+  class ArchiveExtract < Proxy::Util::CommandTask
+    include Util
+
+    SHELL_COMMAND = 'isoinfo'
+
+    def initialize(image_path, file_in_image, dst_path)
+      args = [
+        which(SHELL_COMMAND),
+        # Print information from Rock Ridge extensions
+        '-R',
+        # Filename to read ISO-9660 image from
+        '-i', image_path.to_s,
+        # Extract specified file to stdout
+        '-x', file_in_image.to_s
+      ]
+
+      super(args, nil, dst_path)
+    end
+
+    def start
+      lock = Proxy::FileLock.try_locking(File.join(File.dirname(@output), ".#{File.basename(@output)}.lock"))
+      if lock.nil?
+        false
+      else
+        super do
+          Proxy::FileLock.unlock(lock)
+          File.unlink(lock)
+        end
+      end
+    end
+  end
+end

lib/proxy/util.rb

@@ -13,12 +13,19 @@ class CommandTask
     # stderr is redirected to proxy error log, stdout to proxy debug log
     # command can be either string or array (command + arguments)
     # input is passed into STDIN and must be string
-    def initialize(command, input = nil)
+    # output can be a string containing a file path. If this is the case,
+    # output is not logged but written to this file.
+    def initialize(command, input = nil, output = nil)
       @command = command
       @input = input
+      @output = output
     end
 
     def start(&ensured_block)
+      @output.nil? ? spawn_logging_thread(&ensured_block) : spawn_output_thread(&ensured_block)
+    end
+
+    def spawn_logging_thread(&ensured_block)
       # run the task in its own thread
       @task = Thread.new(@command, @input) do |cmd, input|
         status = nil
@@ -43,6 +50,26 @@ def start(&ensured_block)
       self
     end
 
+    def spawn_output_thread(&ensured_block)
+      # run the task in its own thread
+      @task = Thread.new(@command, @input, @output) do |cmd, input, file|
+        status = nil
+        Open3.pipeline_w(cmd, :out => file.to_s) do |stdin, thr|
+          cmdline_string = Shellwords.escape(cmd.is_a?(Array) ? cmd.join(' ') : cmd)
+          last_thr = thr[-1]
+          logger.info "[#{last_thr.pid}] Started task #{cmdline_string}"
+          stdin.write(input) if input
+          stdin.close
+          # call thr.value to wait for a Process::Status object.
+          status = last_thr.value
+        end
+        status ? status.exitstatus : $CHILD_STATUS
+      ensure
+        yield if block_given?
+      end
+      self
+    end
+
     # wait for the task to finish and get the subprocess return code
     def join
       @task.value

lib/smart_proxy_for_testing.rb

@@ -10,6 +10,7 @@
 require 'proxy/dependency_injection'
 require 'proxy/util'
 require 'proxy/http_download'
+require 'proxy/archive_extract'
 require 'proxy/helpers'
 require 'proxy/memory_store'
 require 'proxy/plugin_validators'

lib/smart_proxy_main.rb

@@ -14,6 +14,7 @@
 require 'proxy/dependency_injection'
 require 'proxy/util'
 require 'proxy/http_download'
+require 'proxy/archive_extract'
 require 'proxy/helpers'
 require 'proxy/memory_store'
 require 'proxy/plugin_validators'

modules/tftp/http_config.ru

@@ -1,5 +1,10 @@
 require 'tftp/tftp_api'
+require 'tftp/tftp_bootimage_api'
 
 map "/tftp" do
   run Proxy::TFTP::Api
 end
+
+map "/tftp/bootimage" do
+  run Proxy::TFTP::BootimageApi
+end

modules/tftp/server.rb

@@ -150,6 +150,43 @@ def pxeconfig_file(mac)
     end
   end
 
+  def self.fetch_boot_image(image_dst, url, files)
+    base_path = Pathname.new(Proxy::TFTP::Plugin.settings.boot_image_root).cleanpath
+    image_path = Pathname.new(File.expand_path(image_dst, base_path)).cleanpath
+
+    # Verify image_dst does not contain ".." (switch folder)
+    if image_path.expand_path.relative_path_from(base_path).to_s.start_with?('..')
+      raise "File to extract from image contains up-directory: #{image_dst}"
+    end
+
+    extr_image_dir = image_path.sub_ext('')
+
+    image_path.parent.mkpath
+    download_task = choose_protocol_and_fetch(url, image_path)
+    # Wait for concurrent processes
+    raise "Cannot download boot image at the moment - is another process downloading it already?" if download_task.is_a?(FalseClass)
+    # wait for download completion
+    download_task.join
+
+    files.each do |file|
+      file_path = Pathname.new file
+      extr_file_path = Pathname.new(File.join(extr_image_dir, file_path)).cleanpath
+
+      # Verify file does not contain ".." (switch folder)
+      if extr_file_path.expand_path.relative_path_from(extr_image_dir).to_s.start_with?('..')
+        raise "File to extract from image contains up-directory: #{file_path}"
+      end
+
+      # Create destination directory
+      extr_file_path.parent.mkpath
+      # extract iso
+      unless File.exist? extr_file_path
+        extract_task = ::Proxy::ArchiveExtract.new(image_path, file_path, extr_file_path).start
+        raise "TFTP image file extraction error: #{file_path}" unless extract_task.join == 0
+      end
+    end
+  end
+
   def self.fetch_boot_file(dst, src)
     filename    = boot_filename(dst, src)
     destination = Pathname.new(File.expand_path(filename, Proxy::TFTP::Plugin.settings.tftproot)).cleanpath

modules/tftp/tftp_api.rb

@@ -34,6 +34,14 @@ def create_default(variant)
       end
     end
 
+    post "/fetch_boot_image" do
+      log_halt(400, "TFTP: Wrong input parameters given.") unless [params[:path], params[:url], params[:files]].all?
+
+      log_halt(500, "TFTP: Failed to fetch boot file: ") do
+        Proxy::TFTP.fetch_boot_image(params[:path], params[:url], params[:files])
+      end
+    end
+
     post "/fetch_boot_file" do
       log_halt(400, "TFTP: Failed to fetch boot file: ") { Proxy::TFTP.fetch_boot_file(params[:prefix], params[:path]) }
     end

modules/tftp/tftp_bootimage_api.rb

@@ -0,0 +1,18 @@
+module Proxy::TFTP
+  class BootimageApi < ::Sinatra::Base
+    helpers ::Proxy::Helpers
+
+    get "/*" do
+      file = Pathname.new(params[:splat].first).cleanpath
+      root = Pathname.new(Proxy::TFTP::Plugin.settings.boot_image_root).expand_path.cleanpath
+      joined_path = File.join(root, file)
+      log_halt(404, "Not found") unless File.exist?(joined_path)
+      real_file = Pathname.new(joined_path).realpath
+      log_halt(403, "Invalid or empty path") unless real_file.fnmatch?("#{root}/**")
+      log_halt(403, "Directory listing not allowed") if File.directory?(real_file)
+      log_halt(503, "Not a regular file") unless File.file?(real_file)
+      send_file real_file
+    end
+  end
+end
+

modules/tftp/tftp_plugin.rb

@@ -6,9 +6,13 @@ class Plugin < ::Proxy::Plugin
 
     default_settings :tftproot => '/var/lib/tftpboot',
                      :tftp_connect_timeout => 10,
-                     :verify_server_cert => true
+                     :verify_server_cert => true,
+                     :boot_image_root => '/var/lib/foreman-proxy/tftp/boot_images'
     validate :verify_server_cert, boolean: true
 
+    # Expose automatic iso handling capability
+    capability -> { which(Proxy::ArchiveExtract.SHELL_COMMAND) ? 'extraction' : nil }
+
     expose_setting :tftp_servername
   end
 end

test/tftp/tftp_api_test.rb

@@ -111,6 +111,12 @@ def test_api_can_fetch_boot_file
     assert last_response.ok?
   end
 
+  def test_api_can_fetch_boot_image
+    Proxy::TFTP.expects(:fetch_boot_image).with('some/image.iso', 'http://localhost/file.iso', ['dir/first', 'dir/second']).returns(true)
+    post "/fetch_boot_image", :path => 'some/image.iso', :url => 'http://localhost/file.iso', :files => ['dir/first', 'dir/second']
+    assert last_response.ok?
+  end
+
   def test_api_can_get_servername
     Proxy::TFTP::Plugin.settings.stubs(:tftp_servername).returns("servername")
     result = get "/serverName"