From 443d48bc258f7599e096f182d25f6572eb72575c Mon Sep 17 00:00:00 2001 From: Thorben Denzer Date: Thu, 2 Nov 2023 15:29:44 +0100 Subject: [PATCH] Fixes #36971 - GUI to allow cloning of Ansible roles from VCS --- lib/smart_proxy_ansible.rb | 2 + lib/smart_proxy_ansible/api.rb | 32 +++++ lib/smart_proxy_ansible/plugin.rb | 57 ++++++++- lib/smart_proxy_ansible/roles_reader.rb | 10 +- lib/smart_proxy_ansible/vcs_cloner.rb | 98 +++++++++++++++ lib/smart_proxy_ansible/vcs_cloner_helper.rb | 42 +++++++ settings.d/ansible.yml.example | 5 + smart_proxy_ansible.gemspec | 2 + test/roles_reader_test.rb | 5 +- test/vcs_cloner_test.rb | 124 +++++++++++++++++++ 10 files changed, 368 insertions(+), 9 deletions(-) create mode 100644 lib/smart_proxy_ansible/vcs_cloner.rb create mode 100644 lib/smart_proxy_ansible/vcs_cloner_helper.rb create mode 100644 test/vcs_cloner_test.rb diff --git a/lib/smart_proxy_ansible.rb b/lib/smart_proxy_ansible.rb index 119a983..1ab0e0b 100644 --- a/lib/smart_proxy_ansible.rb +++ b/lib/smart_proxy_ansible.rb @@ -11,5 +11,7 @@ module Ansible require 'smart_proxy_ansible/playbooks_reader' require 'smart_proxy_ansible/reader_helper' require 'smart_proxy_ansible/variables_extractor' + require 'smart_proxy_ansible/vcs_cloner' + require 'git' end end diff --git a/lib/smart_proxy_ansible/api.rb b/lib/smart_proxy_ansible/api.rb index 67776ee..0a85d1d 100644 --- a/lib/smart_proxy_ansible/api.rb +++ b/lib/smart_proxy_ansible/api.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Proxy module Ansible # API endpoints. Most of the code should be calling other classes, @@ -36,6 +38,36 @@ class Api < Sinatra::Base PlaybooksReader.playbooks(params[:playbooks_names]).to_json end + get '/vcs_clone/repo_information' do + repo_info = VCSCloner.repo_information(params) + status repo_info.status + body repo_info.payload.to_json + end + + get '/vcs_clone/roles' do + get_installed = VCSCloner.list_installed_roles + status get_installed.status + body get_installed.payload.to_json + end + + post '/vcs_clone/roles' do + install = VCSCloner.install(params['repo_info']) + status install.status + body install.payload.to_json + end + + put '/vcs_clone/roles/:role_name' do + update = VCSCloner.update(params['repo_info']) + status update.status + body update.payload.to_json + end + + delete '/vcs_clone/roles/:role_name' do + delete = VCSCloner.delete(params) + status delete.status + body delete.payload.to_json + end + private def extract_variables(role_name) diff --git a/lib/smart_proxy_ansible/plugin.rb b/lib/smart_proxy_ansible/plugin.rb index 9ea4b80..2527f62 100644 --- a/lib/smart_proxy_ansible/plugin.rb +++ b/lib/smart_proxy_ansible/plugin.rb @@ -1,17 +1,64 @@ +# frozen_string_literal: true + module Proxy module Ansible # Calls for the smart-proxy API to register the plugin class Plugin < Proxy::Plugin + + def self.runtime_dir + Pathname(ENV['RUNTIME_DIRECTORY'] || '/run').join(plugin_name.to_s) + end + + def self.state_dir + Pathname(ENV['STATE_DIRECTORY'] || '/var/lib/foreman-proxy').join(plugin_name.to_s) + end + + def self.cache_dir + Pathname(ENV['CACHE_DIRECTORY'] || '/var/cache/foreman-proxy').join(plugin_name.to_s) + end + + def self.logs_dir + Pathname(ENV['LOGS_DIRECTORY'] || '/var/logs/foreman-proxy').join(plugin_name.to_s) + end + + def self.config_dir + Pathname(ENV['CONFIGURATION_DIRECTORY'] || '/etc/foreman-proxy').join(plugin_name.to_s) + end + rackup_path File.expand_path('http_config.ru', __dir__) settings_file 'ansible.yml' plugin :ansible, Proxy::Ansible::VERSION - default_settings :ansible_dir => Dir.home, - :ansible_environment_file => '/etc/foreman-proxy/ansible.env' - # :working_dir => nil + default_settings ansible_dir: Dir.home, + ansible_environment_file: '/etc/foreman-proxy/ansible.env', + vcs_integration: true, + static_roles_paths: %w[/etc/ansible/roles /usr/share/ansible/roles] + + load_programmable_settings do |settings| + mutable_roles_path = settings[:mutable_roles_path] || state_dir.join('roles') + system_roles_path = settings[:static_roles_paths].join(':') + if settings[:vcs_integration] + unless Pathname.new(mutable_roles_path).exist? + raise StandardError, + "#{mutable_roles_path} does not exist. Create it or disable vcs_integration" + end + unless File.writable?(mutable_roles_path) + raise StandardError, + "#{mutable_roles_path} is not writable. Check permissions or disable vcs_integration" + end + + settings[:all_roles_path] = "#{mutable_roles_path}:#{system_roles_path}" + settings[:mutable_roles_path] = mutable_roles_path + + else + settings[:all_roles_path] = system_roles_path + end + settings + end load_classes ::Proxy::Ansible::ConfigurationLoader - load_validators :validate_settings => ::Proxy::Ansible::ValidateSettings - validate :validate!, :validate_settings => nil + load_validators validate_settings: ::Proxy::Ansible::ValidateSettings + capability :vcs_clone + validate :validate!, validate_settings: nil end end end diff --git a/lib/smart_proxy_ansible/roles_reader.rb b/lib/smart_proxy_ansible/roles_reader.rb index b690caf..07bb7e5 100644 --- a/lib/smart_proxy_ansible/roles_reader.rb +++ b/lib/smart_proxy_ansible/roles_reader.rb @@ -5,16 +5,20 @@ module Ansible # Implements the logic needed to read the roles and associated information class RolesReader class << self - DEFAULT_ROLES_PATH = '/etc/ansible/roles:/usr/share/ansible/roles'.freeze def list_roles - roles = roles_path.split(':').map { |path| read_roles(path) }.flatten + all_roles = roles_path.split(':').map { |path| read_roles(path) }.flatten + roles = Set.new(all_roles).to_a collection_roles = ReaderHelper.collections_paths.split(':').map { |path| read_collection_roles(path) }.flatten roles + collection_roles end + def roles_path_from_settings + Proxy::Ansible::Plugin.settings[:all_roles_path] + end + def roles_path - ReaderHelper.config_path(ReaderHelper.path_from_config('roles_path'), DEFAULT_ROLES_PATH) + ReaderHelper.config_path(ReaderHelper.path_from_config('roles_path'), roles_path_from_settings) end def logger diff --git a/lib/smart_proxy_ansible/vcs_cloner.rb b/lib/smart_proxy_ansible/vcs_cloner.rb new file mode 100644 index 0000000..68a9e93 --- /dev/null +++ b/lib/smart_proxy_ansible/vcs_cloner.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require_relative 'vcs_cloner_helper' +require 'net/http' + +module Proxy + include ::Proxy::Log + + module Ansible + Response = Struct.new(:status, :payload) + + # VCSCloner. This class performs cloning and updating of Ansible-Roles sourced from Git + class VCSCloner + class << self + # Queries metadata about a given repository. + # Requires parameter "vcs_url" + # Returns 200 and the data if the query was successful + # Returns 400 if a parameter is unfulfilled or invalid repo-info was provided + def repo_information(payload) + return Response.new(400, 'Check parameters') unless payload.key? 'vcs_url' + + vcs_url = payload['vcs_url'] + remote = Git.ls_remote(vcs_url).slice('head', 'branches', 'tags') + remote['vcs_url'] = vcs_url + Response.new(200, remote) + rescue Git::GitExecuteError => e + Response.new(400, "Git Error: #{e}") + end + + # Returns an array of installed roles + # Uses RolesReader.list_roles + def list_installed_roles + Response.new(200, RolesReader.list_roles) + end + + # Clones a new role from the provided information. + # Requires hash with keys "vcs_url", "name" and "ref" + # Returns 201 if a role was created + # Returns 400 if a parameter is unfulfilled or invalid repo-info was provided + # Returns 409 if a role with "name" already exists + def install(repo_info) + return Response.new(400, 'Check parameters') unless VcsClonerHelper.correct_repo_info(repo_info) + + if VcsClonerHelper.role_exists repo_info['name'] + return Response.new(409, + "Role \"#{repo_info['name']}\" already exists.") + end + + begin VcsClonerHelper.install_role repo_info + Response.new(201, "Role \"#{repo_info['name']}\" has been created.") + rescue Git::GitExecuteError => e + Response.new(400, "Git Error: #{e}") + end + end + + # Updates a role with the provided information. + # Installs a role if it does not yet exist + # Requires hash with keys "vcs_url", "name" and "ref" + # Returns 200 if a role was updated + # Returns 201 if a role was created + # Returns 400 if a parameter is unfulfilled or invalid repo-info was provided + def update(repo_info) + return Response.new(400, 'Check parameters') unless VcsClonerHelper.correct_repo_info repo_info + + begin + if VcsClonerHelper.role_exists repo_info['name'] + VcsClonerHelper.update_role repo_info + Response.new(200, "Role \"#{repo_info['name']}\" has been updated.") + else + VcsClonerHelper.install_role repo_info + Response.new(201, "Role \"#{repo_info['name']}\" has been created.") + end + rescue Git::GitExecuteError => e + Response.new(400, "Git Error: #{e}") + end + end + + # Deletes a role with the given name. + # Installs a role if it does not yet exist + # Requires parameter role_name + # Returns 200 if a role was deleted / never existed + # Returns 400 if a parameter is unfulfilled + def delete(payload) + return Response.new(400, 'Check parameters') unless payload.key? 'role_name' + + role_name = payload['role_name'] + unless VcsClonerHelper.role_exists role_name + return Response.new(200, + "Role \"#{role_name}\" does not exist. Request ignored.") + end + + VcsClonerHelper.delete_role role_name + Response.new(200, "Role \"#{role_name}\" has been deleted.") + end + end + end + end +end diff --git a/lib/smart_proxy_ansible/vcs_cloner_helper.rb b/lib/smart_proxy_ansible/vcs_cloner_helper.rb new file mode 100644 index 0000000..e4ccbda --- /dev/null +++ b/lib/smart_proxy_ansible/vcs_cloner_helper.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Proxy + module Ansible + # Implements VCS-Cloning logic and helper functions + class VcsClonerHelper + class << self + def repo_path(role_name) + @base_path ||= Pathname(Proxy::Ansible::Plugin.settings[:mutable_roles_path]) + @base_path.join(role_name) + end + + def correct_repo_info(repo_info) + %w[vcs_url name ref].all? { |param| repo_info.key?(param) } + end + + def role_exists(role_name) + repo_path(role_name).exist? + end + + def install_role(repo_info) + git = Git.init(repo_path(repo_info['name'])) + git.add_remote('origin', repo_info['vcs_url']) + git.fetch + git.checkout(repo_info['ref']) + end + + def update_role(repo_info) + git = Git.open(repo_path(repo_info['name'])) + git.remove_remote('origin') + git.add_remote('origin', repo_info['vcs_url']) + git.fetch + git.checkout(repo_info['ref']) + end + + def delete_role(role_name) + FileUtils.rm_r repo_path(role_name) + end + end + end + end +end diff --git a/settings.d/ansible.yml.example b/settings.d/ansible.yml.example index c702ad3..4b42b75 100644 --- a/settings.d/ansible.yml.example +++ b/settings.d/ansible.yml.example @@ -1,4 +1,9 @@ --- :enabled: true :working_dir: '~/.foreman-ansible' +:vcs_integration: true +:static_roles_paths: [ + '/etc/ansible/roles', + '/usr/share/ansible/roles', +] # :ansible_environment_file: /etc/foreman-proxy/ansible.env diff --git a/smart_proxy_ansible.gemspec b/smart_proxy_ansible.gemspec index 431a6c4..bcd6695 100644 --- a/smart_proxy_ansible.gemspec +++ b/smart_proxy_ansible.gemspec @@ -27,6 +27,8 @@ Gem::Specification.new do |gem| gem.license = 'GPL-3.0' gem.required_ruby_version = '>= 2.5' + gem.add_dependency 'git', '~> 1.0' + gem.add_development_dependency 'rake', '~> 13.0' gem.add_development_dependency('mocha', '~> 1') gem.add_development_dependency('webmock', '~> 3') diff --git a/test/roles_reader_test.rb b/test/roles_reader_test.rb index f3594cf..18258f1 100644 --- a/test/roles_reader_test.rb +++ b/test/roles_reader_test.rb @@ -7,7 +7,7 @@ # this class simply reads roles from its path in ansible.cfg class RolesReaderTest < Minitest::Test CONFIG_PATH = ::Proxy::Ansible::ReaderHelper.singleton_class::DEFAULT_CONFIG_FILE - ROLES_PATH = ::Proxy::Ansible::RolesReader.singleton_class::DEFAULT_ROLES_PATH + ROLES_PATH = '/etc/ansible/roles:/usr/share/ansible/roles' COLLECTIONS_PATHS = ::Proxy::Ansible::ReaderHelper.singleton_class::DEFAULT_COLLECTIONS_PATHS def self.expect_content_config(ansible_cfg_content) @@ -17,6 +17,7 @@ def self.expect_content_config(ansible_cfg_content) describe '#roles_path' do test 'detects commented roles_path' do + Proxy::Ansible::RolesReader.stubs(:roles_path_from_settings).returns('/etc/ansible/roles:/usr/share/ansible/roles') RolesReaderTest.expect_content_config ['#roles_path = thisiscommented!'] assert_equal(ROLES_PATH, Proxy::Ansible::RolesReader.roles_path) @@ -90,6 +91,7 @@ def setup describe 'with unreadable config' do test 'handles "No such file or directory" by using defaults' do + Proxy::Ansible::RolesReader.stubs(:roles_path_from_settings).returns('/etc/ansible/roles:/usr/share/ansible/roles') File.expects(:readlines).times(2).with(CONFIG_PATH).raises(Errno::ENOENT) ROLES_PATH.split(':').map do |path| @@ -105,6 +107,7 @@ def setup end test 'handles "Permission denied" by using defaults' do + Proxy::Ansible::RolesReader.stubs(:roles_path_from_settings).returns('/etc/ansible/roles:/usr/share/ansible/roles') File.expects(:readlines).times(2).with(CONFIG_PATH).raises(Errno::EACCES) ROLES_PATH.split(':').map do |path| diff --git a/test/vcs_cloner_test.rb b/test/vcs_cloner_test.rb new file mode 100644 index 0000000..472e85d --- /dev/null +++ b/test/vcs_cloner_test.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'git' + +require_relative '../lib/smart_proxy_ansible/vcs_cloner' +require_relative '../lib/smart_proxy_ansible/roles_reader' + +# Tests VCSCloner class +class VcsClonerTest < Minitest::Test + Response = Proxy::Ansible::Response + + describe '#repo_information' do + payload = { + 'vcs_url' => 'https://github.com/theforeman/smart_proxy_ansible.git' + } + demo_info = { + 'head' => {}, + 'branches' => {}, + 'tags' => {} + } + test 'requests repo information' do + Git.stubs(:ls_remote).returns(demo_info) + response = Proxy::Ansible::VCSCloner.repo_information payload + assert_equal Response.new(200, payload.merge(demo_info)), response + end + + test 'handles a missing parameter correctly' do + response = Proxy::Ansible::VCSCloner.repo_information({}) + assert_equal Response.new(400, 'Check parameters'), response + end + end + describe '#list_installed_roles' do + demo_roles = %w[role1 role2 role3] + test 'correctly lists installed roles' do + Proxy::Ansible::RolesReader.stubs(:list_roles).returns(demo_roles) + response = Proxy::Ansible::VCSCloner.list_installed_roles + assert_equal Response.new(200, demo_roles), response + end + end + describe '#install' do + demo_repo_info = { + 'vcs_url' => 'https://some.git.url', + 'name' => 'best.role.ever', + 'ref' => 'master' + } + test 'installs a role' do + Proxy::Ansible::VcsClonerHelper.stubs(:role_exists).returns(false) + Proxy::Ansible::VcsClonerHelper.stubs(:install_role).returns(true) + response = Proxy::Ansible::VCSCloner.install demo_repo_info + assert_equal Response.new(201, 'Role "best.role.ever" has been created.'), response + end + + test 'handles a conflict properly' do + Proxy::Ansible::VcsClonerHelper.stubs(:role_exists).returns(true) + response = Proxy::Ansible::VCSCloner.install demo_repo_info + assert_equal Response.new(409, 'Role "best.role.ever" already exists.'), response + end + + test 'handles missing parameter properly' do + response = Proxy::Ansible::VCSCloner.install({ + 'name' => 'best.role.ever', + 'ref' => 'master' + }) + assert_equal Response.new(400, 'Check parameters'), response + end + test 'handles Git error' do + Proxy::Ansible::VcsClonerHelper.stubs(:install_role).raises(Git::GitExecuteError.new) + Proxy::Ansible::VcsClonerHelper.stubs(:role_exists).returns(false) + response = Proxy::Ansible::VCSCloner.install demo_repo_info + assert_equal Response.new(400, 'Git Error: Git::GitExecuteError'), response + end + end + describe '#update' do + demo_repo_info = { + 'vcs_url' => 'https://some.git.url', + 'name' => 'best.role.ever', + 'ref' => 'master' + } + test 'updates a role' do + Proxy::Ansible::VcsClonerHelper.stubs(:role_exists).returns(true) + Proxy::Ansible::VcsClonerHelper.stubs(:update_role).returns(true) + response = Proxy::Ansible::VCSCloner.update demo_repo_info + assert_equal Response.new(200, 'Role "best.role.ever" has been updated.'), response + end + test 'installs a role' do + Proxy::Ansible::VcsClonerHelper.stubs(:role_exists).returns(false) + Proxy::Ansible::VcsClonerHelper.stubs(:install_role).returns(true) + response = Proxy::Ansible::VCSCloner.update demo_repo_info + assert_equal Response.new(201, 'Role "best.role.ever" has been created.'), response + end + test 'handles missing parameter properly' do + response = Proxy::Ansible::VCSCloner.update({ + 'name' => 'best.role.ever', + 'ref' => 'master' + }) + assert_equal Response.new(400, 'Check parameters'), response + end + test 'handles Git error' do + Proxy::Ansible::VcsClonerHelper.stubs(:role_exists).returns(true) + Proxy::Ansible::VcsClonerHelper.stubs(:update_role).raises(Git::GitExecuteError.new) + response = Proxy::Ansible::VCSCloner.update demo_repo_info + assert_equal Response.new(400, 'Git Error: Git::GitExecuteError'), response + end + end + describe '#delete' do + test 'deletes a role' do + Proxy::Ansible::VcsClonerHelper.stubs(:role_exists).returns(true) + Proxy::Ansible::VcsClonerHelper.stubs(:delete_role).returns(true) + response = Proxy::Ansible::VCSCloner.delete({ 'role_name' => 'best.role.ever' }) + assert_equal Response.new(200, 'Role "best.role.ever" has been deleted.'), response + end + + test 'skips deleting a role' do + Proxy::Ansible::VcsClonerHelper.stubs(:role_exists).returns(false) + response = Proxy::Ansible::VCSCloner.delete({ 'role_name' => 'best.role.ever' }) + assert_equal Response.new(200, 'Role "best.role.ever" does not exist. Request ignored.'), response + end + test 'handles missing parameter properly' do + response = Proxy::Ansible::VCSCloner.delete({}) + assert_equal Response.new(400, 'Check parameters'), response + end + end +end