Fixes #36971 - GUI to allow cloning of Ansible roles from VCS

lib/smart_proxy_ansible.rb

@@ -11,5 +11,7 @@ module Ansible
     require 'smart_proxy_ansible/playbooks_reader'
     require 'smart_proxy_ansible/reader_helper'
     require 'smart_proxy_ansible/variables_extractor'
+    require 'smart_proxy_ansible/vcs_cloner'
+    require 'git'
   end
 end

lib/smart_proxy_ansible/api.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Proxy
   module Ansible
     # API endpoints. Most of the code should be calling other classes,
@@ -36,6 +38,36 @@ class Api < Sinatra::Base
         PlaybooksReader.playbooks(params[:playbooks_names]).to_json
       end
 
+      get '/vcs_clone/repo_information' do
+        repo_info = VCSCloner.repo_information(params)
+        status repo_info.status
+        body repo_info.payload.to_json
+      end
+
+      get '/vcs_clone/roles' do
+        get_installed = VCSCloner.list_installed_roles
+        status get_installed.status
+        body get_installed.payload.to_json
+      end
+
+      post '/vcs_clone/roles' do
+        install = VCSCloner.install(params['repo_info'])
+        status install.status
+        body install.payload.to_json
+      end
+
+      put '/vcs_clone/roles/:role_name' do
+        update = VCSCloner.update(params['repo_info'])
+        status update.status
+        body update.payload.to_json
+      end
+
+      delete '/vcs_clone/roles/:role_name' do
+        delete = VCSCloner.delete(params)
+        status delete.status
+        body delete.payload.to_json
+      end
+
       private
 
       def extract_variables(role_name)

lib/smart_proxy_ansible/plugin.rb

@@ -1,17 +1,64 @@
+# frozen_string_literal: true
+
 module Proxy
   module Ansible
     # Calls for the smart-proxy API to register the plugin
     class Plugin < Proxy::Plugin
+
+      def self.runtime_dir
+        Pathname(ENV['RUNTIME_DIRECTORY'] || '/run').join(plugin_name.to_s)
+      end
+
+      def self.state_dir
+        Pathname(ENV['STATE_DIRECTORY'] || '/var/lib/foreman-proxy').join(plugin_name.to_s)
+      end
+
+      def self.cache_dir
+        Pathname(ENV['CACHE_DIRECTORY'] || '/var/cache/foreman-proxy').join(plugin_name.to_s)
+      end
+
+      def self.logs_dir
+        Pathname(ENV['LOGS_DIRECTORY'] || '/var/logs/foreman-proxy').join(plugin_name.to_s)
+      end
+
+      def self.config_dir
+        Pathname(ENV['CONFIGURATION_DIRECTORY'] || '/etc/foreman-proxy').join(plugin_name.to_s)
+      end
+
       rackup_path File.expand_path('http_config.ru', __dir__)
       settings_file 'ansible.yml'
       plugin :ansible, Proxy::Ansible::VERSION
-      default_settings :ansible_dir => Dir.home,
-                       :ansible_environment_file => '/etc/foreman-proxy/ansible.env'
-                       # :working_dir => nil
+      default_settings ansible_dir: Dir.home,
+                       ansible_environment_file: '/etc/foreman-proxy/ansible.env',
+                       vcs_integration: true,
+                       static_roles_paths: %w[/etc/ansible/roles /usr/share/ansible/roles]
+
+      load_programmable_settings do |settings|
+        mutable_roles_path = settings[:mutable_roles_path] || state_dir.join('roles')
+        system_roles_path = settings[:static_roles_paths].join(':')
+        if settings[:vcs_integration]
+          unless Pathname.new(mutable_roles_path).exist?
+            raise StandardError,
+                  "#{mutable_roles_path} does not exist. Create it or disable vcs_integration"
+          end
+          unless File.writable?(mutable_roles_path)
+            raise StandardError,
+                  "#{mutable_roles_path} is not writable. Check permissions or disable vcs_integration"
+          end
+
+          settings[:all_roles_path] = "#{mutable_roles_path}:#{system_roles_path}"
+          settings[:mutable_roles_path] = mutable_roles_path
+
+        else
+          settings[:all_roles_path] = system_roles_path
+        end
+        settings
+      end
 
       load_classes ::Proxy::Ansible::ConfigurationLoader
-      load_validators :validate_settings => ::Proxy::Ansible::ValidateSettings
-      validate :validate!, :validate_settings => nil
+      load_validators validate_settings: ::Proxy::Ansible::ValidateSettings
+      capability :vcs_clone
+      validate :validate!, validate_settings: nil
     end
   end
 end

lib/smart_proxy_ansible/roles_reader.rb

@@ -5,16 +5,20 @@ module Ansible
     # Implements the logic needed to read the roles and associated information
     class RolesReader
       class << self
-        DEFAULT_ROLES_PATH = '/etc/ansible/roles:/usr/share/ansible/roles'.freeze
 
         def list_roles
-          roles = roles_path.split(':').map { |path| read_roles(path) }.flatten
+          all_roles = roles_path.split(':').map { |path| read_roles(path) }.flatten
+          roles = Set.new(all_roles).to_a
           collection_roles = ReaderHelper.collections_paths.split(':').map { |path| read_collection_roles(path) }.flatten
           roles + collection_roles
         end
 
+        def roles_path_from_settings
+          Proxy::Ansible::Plugin.settings[:all_roles_path]
+        end
+
         def roles_path
-          ReaderHelper.config_path(ReaderHelper.path_from_config('roles_path'), DEFAULT_ROLES_PATH)
+          ReaderHelper.config_path(ReaderHelper.path_from_config('roles_path'), roles_path_from_settings)
         end
 
         def logger

lib/smart_proxy_ansible/vcs_cloner.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative 'vcs_cloner_helper'
+require 'net/http'
+
+module Proxy
+  include ::Proxy::Log
+
+  module Ansible
+    Response = Struct.new(:status, :payload)
+
+    # VCSCloner. This class performs cloning and updating of Ansible-Roles sourced from Git
+    class VCSCloner
+      class << self
+        # Queries metadata about a given repository.
+        # Requires parameter "vcs_url"
+        # Returns 200 and the data if the query was successful
+        # Returns 400 if a parameter is unfulfilled or invalid repo-info was provided
+        def repo_information(payload)
+          return Response.new(400, 'Check parameters') unless payload.key? 'vcs_url'
+
+          vcs_url = payload['vcs_url']
+          remote = Git.ls_remote(vcs_url).slice('head', 'branches', 'tags')
+          remote['vcs_url'] = vcs_url
+          Response.new(200, remote)
+        rescue Git::GitExecuteError => e
+          Response.new(400, "Git Error: #{e}")
+        end
+
+        # Returns an array of installed roles
+        # Uses RolesReader.list_roles
+        def list_installed_roles
+          Response.new(200, RolesReader.list_roles)
+        end
+
+        # Clones a new role from the provided information.
+        # Requires hash with keys "vcs_url", "name" and "ref"
+        # Returns 201 if a role was created
+        # Returns 400 if a parameter is unfulfilled or invalid repo-info was provided
+        # Returns 409 if a role with "name" already exists
+        def install(repo_info)
+          return Response.new(400, 'Check parameters') unless VcsClonerHelper.correct_repo_info(repo_info)
+
+          if VcsClonerHelper.role_exists repo_info['name']
+            return Response.new(409,
+                                "Role \"#{repo_info['name']}\" already exists.")
+          end
+
+          begin VcsClonerHelper.install_role repo_info
+                Response.new(201, "Role \"#{repo_info['name']}\" has been created.")
+          rescue Git::GitExecuteError => e
+            Response.new(400, "Git Error: #{e}")
+          end
+        end
+
+        # Updates a role with the provided information.
+        # Installs a role if it does not yet exist
+        # Requires hash with keys "vcs_url", "name" and "ref"
+        # Returns 200 if a role was updated
+        # Returns 201 if a role was created
+        # Returns 400 if a parameter is unfulfilled or invalid repo-info was provided
+        def update(repo_info)
+          return Response.new(400, 'Check parameters') unless VcsClonerHelper.correct_repo_info repo_info
+
+          begin
+            if VcsClonerHelper.role_exists repo_info['name']
+              VcsClonerHelper.update_role repo_info
+              Response.new(200, "Role \"#{repo_info['name']}\" has been updated.")
+            else
+              VcsClonerHelper.install_role repo_info
+              Response.new(201, "Role \"#{repo_info['name']}\" has been created.")
+            end
+          rescue Git::GitExecuteError => e
+            Response.new(400, "Git Error: #{e}")
+          end
+        end
+
+        # Deletes a role with the given name.
+        # Installs a role if it does not yet exist
+        # Requires parameter role_name
+        # Returns 200 if a role was deleted / never existed
+        # Returns 400 if a parameter is unfulfilled
+        def delete(payload)
+          return Response.new(400, 'Check parameters') unless payload.key? 'role_name'
+
+          role_name = payload['role_name']
+          unless VcsClonerHelper.role_exists role_name
+            return Response.new(200,
+                                "Role \"#{role_name}\" does not exist. Request ignored.")
+          end
+
+          VcsClonerHelper.delete_role role_name
+          Response.new(200, "Role \"#{role_name}\" has been deleted.")
+        end
+      end
+    end
+  end
+end

lib/smart_proxy_ansible/vcs_cloner_helper.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Proxy
+  module Ansible
+    # Implements VCS-Cloning logic and helper functions
+    class VcsClonerHelper
+      class << self
+        def repo_path(role_name)
+          @base_path ||= Pathname(Proxy::Ansible::Plugin.settings[:mutable_roles_path])
+          @base_path.join(role_name)
+        end
+
+        def correct_repo_info(repo_info)
+          %w[vcs_url name ref].all? { |param| repo_info.key?(param) }
+        end
+
+        def role_exists(role_name)
+          repo_path(role_name).exist?
+        end
+
+        def install_role(repo_info)
+          git = Git.init(repo_path(repo_info['name']))
+          git.add_remote('origin', repo_info['vcs_url'])
+          git.fetch
+          git.checkout(repo_info['ref'])
+        end
+
+        def update_role(repo_info)
+          git = Git.open(repo_path(repo_info['name']))
+          git.remove_remote('origin')
+          git.add_remote('origin', repo_info['vcs_url'])
+          git.fetch
+          git.checkout(repo_info['ref'])
+        end
+
+        def delete_role(role_name)
+          FileUtils.rm_r repo_path(role_name)
+        end
+      end
+    end
+  end
+end

settings.d/ansible.yml.example

@@ -1,4 +1,9 @@
 ---
 :enabled: true
 :working_dir: '~/.foreman-ansible'
+:vcs_integration: true
+:static_roles_paths: [
+  '/etc/ansible/roles',
+  '/usr/share/ansible/roles',
+]
 # :ansible_environment_file: /etc/foreman-proxy/ansible.env

smart_proxy_ansible.gemspec

@@ -27,6 +27,8 @@ Gem::Specification.new do |gem|
   gem.license = 'GPL-3.0'
   gem.required_ruby_version = '>= 2.5'
 
+  gem.add_dependency 'git', '~> 1.0'
+
   gem.add_development_dependency 'rake', '~> 13.0'
   gem.add_development_dependency('mocha', '~> 1')
   gem.add_development_dependency('webmock', '~> 3')

test/roles_reader_test.rb

@@ -7,7 +7,7 @@
 # this class simply reads roles from its path in ansible.cfg
 class RolesReaderTest < Minitest::Test
   CONFIG_PATH = ::Proxy::Ansible::ReaderHelper.singleton_class::DEFAULT_CONFIG_FILE
-  ROLES_PATH = ::Proxy::Ansible::RolesReader.singleton_class::DEFAULT_ROLES_PATH
+  ROLES_PATH = '/etc/ansible/roles:/usr/share/ansible/roles'
   COLLECTIONS_PATHS = ::Proxy::Ansible::ReaderHelper.singleton_class::DEFAULT_COLLECTIONS_PATHS
 
   def self.expect_content_config(ansible_cfg_content)
@@ -17,6 +17,7 @@ def self.expect_content_config(ansible_cfg_content)
 
   describe '#roles_path' do
     test 'detects commented roles_path' do
+      Proxy::Ansible::RolesReader.stubs(:roles_path_from_settings).returns('/etc/ansible/roles:/usr/share/ansible/roles')
       RolesReaderTest.expect_content_config ['#roles_path = thisiscommented!']
       assert_equal(ROLES_PATH,
                    Proxy::Ansible::RolesReader.roles_path)
@@ -90,6 +91,7 @@ def setup
 
     describe 'with unreadable config' do
       test 'handles "No such file or directory" by using defaults' do
+        Proxy::Ansible::RolesReader.stubs(:roles_path_from_settings).returns('/etc/ansible/roles:/usr/share/ansible/roles')
         File.expects(:readlines).times(2).with(CONFIG_PATH).raises(Errno::ENOENT)
 
         ROLES_PATH.split(':').map do |path|
@@ -105,6 +107,7 @@ def setup
       end
 
       test 'handles "Permission denied" by using defaults' do
+        Proxy::Ansible::RolesReader.stubs(:roles_path_from_settings).returns('/etc/ansible/roles:/usr/share/ansible/roles')
         File.expects(:readlines).times(2).with(CONFIG_PATH).raises(Errno::EACCES)
 
         ROLES_PATH.split(':').map do |path|