From 635511750ba3b1238018b7f64ef7bc2709bf89e7 Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Mon, 24 Feb 2020 14:46:24 +0100 Subject: [PATCH 1/9] Add sshkey option for git based formula dependencies. --- lib/kitchen/provisioner/dependencies.erb | 12 +++++++++--- lib/kitchen/provisioner/formula-fetch.sh | 9 ++++++++- lib/kitchen/provisioner/known_hosts | 5 +++++ lib/kitchen/provisioner/salt_solo.rb | 23 +++++++++++++++++++---- 4 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 lib/kitchen/provisioner/known_hosts diff --git a/lib/kitchen/provisioner/dependencies.erb b/lib/kitchen/provisioner/dependencies.erb index 9f3df5bb..81e5fd47 100644 --- a/lib/kitchen/provisioner/dependencies.erb +++ b/lib/kitchen/provisioner/dependencies.erb @@ -63,9 +63,15 @@ def install_dependencies if formula.key?(:repo) case formula[:repo] when 'git' - script += <<-INSTALL - fetchGitFormula #{formula[:source]} "#{formula[:name]}" "#{formula[:branch] || 'master'}" - INSTALL + if formula[:sshkey].nil? + script += <<-INSTALL + fetchGitFormula #{formula[:source]} "#{formula[:name]}" "#{formula[:branch] || 'master'}" + INSTALL + else + script += <<-INSTALL + fetchGitFormula #{formula[:source]} "#{formula[:name]}" "#{formula[:branch] || 'master'}" "#{config[:root_path]}/#{config[:ssh_home]}/#{File.basename(formula[:sshkey])}" + INSTALL + end when 'spm' if formula[:package].nil? script += <<-INSTALL diff --git a/lib/kitchen/provisioner/formula-fetch.sh b/lib/kitchen/provisioner/formula-fetch.sh index abefdd9d..60d5abb5 100755 --- a/lib/kitchen/provisioner/formula-fetch.sh +++ b/lib/kitchen/provisioner/formula-fetch.sh @@ -9,12 +9,12 @@ # GIT_FORMULAS_PATH=/usr/share/salt-formulas/env/_formulas # xargs -n1 ./formula-fetch.sh < dependencies.txt - # Parse git dependencies from metadata.yml # $1 - path to /metadata.yml # sample to output: # https://github.com/salt-formulas/salt-formula-git git # https://github.com/salt-formulas/salt-formula-salt salt + function fetchDependencies() { METADATA="$1"; grep -E "^dependencies:" "$METADATA" >/dev/null || return 0 @@ -30,15 +30,22 @@ function fetchDependencies() { # $1 - formula git repo url # $2 - formula name (optional) # $3 - branch (optional) +# $4 - path to deploykey function fetchGitFormula() { test -n "${FETCHED}" || declare -a FETCHED=() export GIT_FORMULAS_PATH=${GIT_FORMULAS_PATH:-/usr/share/salt-formulas/env/_formulas} + if [ $4 != "NULL" ] + then + sshbin=$(which ssh) + export GIT_SSH_COMMAND="${sshbin} -o UserKnownHostsFile=/tmp/kitchen/ssh/known_hosts -o StrictHostKeyChecking=no -i ${4}" + fi mkdir -p "$GIT_FORMULAS_PATH" if [ -n "$1" ]; then source="$1" name="$2" test -n "$name" || name="${source//*salt-formula-}" test -z "$3" && branch=master || branch=$3 + if ! [[ "${FETCHED[*]}" =~ $name ]]; then # dependency not yet fetched echo "Fetching: $name" if test -e "$GIT_FORMULAS_PATH/$name"; then diff --git a/lib/kitchen/provisioner/known_hosts b/lib/kitchen/provisioner/known_hosts new file mode 100644 index 00000000..6295cde4 --- /dev/null +++ b/lib/kitchen/provisioner/known_hosts @@ -0,0 +1,5 @@ +github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== +bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== +gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY= +gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9 +gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf diff --git a/lib/kitchen/provisioner/salt_solo.rb b/lib/kitchen/provisioner/salt_solo.rb index ba6f134a..aaffa7cc 100644 --- a/lib/kitchen/provisioner/salt_solo.rb +++ b/lib/kitchen/provisioner/salt_solo.rb @@ -77,10 +77,11 @@ class SaltSolo < Base salt_spm_root: '/srv/spm', salt_state_top: '/srv/salt/top.sls', salt_version: 'latest', - salt_yum_repo_key: 'https://repo.saltproject.io/yum/redhat/$releasever/$basearch/archive/%s/SALTSTACK-GPG-KEY.pub', - salt_yum_repo_latest: 'https://repo.saltproject.io/yum/redhat/salt-repo-latest-2.el7.noarch.rpm', - salt_yum_repo: 'https://repo.saltproject.io/yum/redhat/$releasever/$basearch/archive/%s', - salt_yum_rpm_key: 'https://repo.saltproject.io/yum/redhat/7/x86_64/archive/%s/SALTSTACK-GPG-KEY.pub', + salt_yum_repo_key: 'https://repo.saltstack.com/yum/redhat/$releasever/$basearch/archive/%s/SALTSTACK-GPG-KEY.pub', + salt_yum_repo_latest: 'https://repo.saltstack.com/yum/redhat/salt-repo-latest-2.el7.noarch.rpm', + salt_yum_repo: 'https://repo.saltstack.com/yum/redhat/$releasever/$basearch/archive/%s', + salt_yum_rpm_key: 'https://repo.saltstack.com/yum/redhat/7/x86_64/archive/%s/SALTSTACK-GPG-KEY.pub', + ssh_home: '/ssh/', state_collection: false, state_top_from_file: false, state_top: {}, @@ -422,6 +423,7 @@ def prepare_grains end def prepare_dependencies +<<<<<<< HEAD # Dependency scripts are bash scripts only # Copying them clobbers the kitchen temp directory # with a file named `kitchen`. If adding Windows @@ -429,6 +431,19 @@ def prepare_dependencies # sub-directory return if windows_os? +======= + # Write ssh known_hosts + write_raw_file(File.join(sandbox_path, config[:ssh_home], "known_hosts"), File.read(File.expand_path("../known_hosts", __FILE__))) + # Write git deploy keys. + config[:dependencies].each do |dependency| + unless dependency[:sshkey].nil? + outfile = File.join(sandbox_path, config[:ssh_home], File.basename(dependency[:sshkey])) + contents = File.read(File.expand_path(dependency[:sshkey])) + info("Copying #{dependency[:sshkey]} to #{outfile}") + write_raw_file(outfile, contents) + end + end +>>>>>>> Add sshkey option for git based formula dependencies. # upload scripts sandbox_scripts_path = File.join(sandbox_path, config[:salt_config], 'scripts') info("Preparing scripts into #{config[:salt_config]}/scripts") From d6f73bca86d765d2312139c33afdbba4fb2936f7 Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Tue, 25 Feb 2020 11:39:06 +0100 Subject: [PATCH 2/9] Use `if xxx.present?` instead of `unless xxx.nil?` --- lib/kitchen/provisioner/salt_solo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kitchen/provisioner/salt_solo.rb b/lib/kitchen/provisioner/salt_solo.rb index aaffa7cc..af0bb9ad 100644 --- a/lib/kitchen/provisioner/salt_solo.rb +++ b/lib/kitchen/provisioner/salt_solo.rb @@ -436,7 +436,7 @@ def prepare_dependencies write_raw_file(File.join(sandbox_path, config[:ssh_home], "known_hosts"), File.read(File.expand_path("../known_hosts", __FILE__))) # Write git deploy keys. config[:dependencies].each do |dependency| - unless dependency[:sshkey].nil? + if dependency[:sshkey].present? outfile = File.join(sandbox_path, config[:ssh_home], File.basename(dependency[:sshkey])) contents = File.read(File.expand_path(dependency[:sshkey])) info("Copying #{dependency[:sshkey]} to #{outfile}") From a5c3008050165551d7376a706a3e30d0019d43da Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Tue, 25 Feb 2020 20:42:47 +0100 Subject: [PATCH 3/9] add support for git <2.3, fixed various bugs, clean up. --- lib/kitchen/provisioner/dependencies.erb | 14 +++++++++--- lib/kitchen/provisioner/formula-fetch.sh | 7 ++++-- lib/kitchen/provisioner/git_ssh.sh | 3 +++ lib/kitchen/provisioner/salt_solo.rb | 28 ++++++++++++++++++------ 4 files changed, 40 insertions(+), 12 deletions(-) create mode 100755 lib/kitchen/provisioner/git_ssh.sh diff --git a/lib/kitchen/provisioner/dependencies.erb b/lib/kitchen/provisioner/dependencies.erb index 81e5fd47..babacd99 100644 --- a/lib/kitchen/provisioner/dependencies.erb +++ b/lib/kitchen/provisioner/dependencies.erb @@ -63,13 +63,21 @@ def install_dependencies if formula.key?(:repo) case formula[:repo] when 'git' - if formula[:sshkey].nil? + if formula[:source].start_with?("http") script += <<-INSTALL - fetchGitFormula #{formula[:source]} "#{formula[:name]}" "#{formula[:branch] || 'master'}" + fetchGitFormula #{formula[:source]} "#{formula[:name]}" "#{formula[:branch] || 'master'}" INSTALL else + if formula[:ssh_key].nil? and config[:ssh_key].nil? + raise "No ssh_key specified for #{formula[:source]}" + end + if formula[:ssh_key].nil? + ssh_key = config[:root_path] + config[:ssh_home] + "/" + File.basename(config[:ssh_key]) + else + ssh_key = config[:root_path] + config[:ssh_home] + "/" + File.basename(formula[:ssh_key]) + end script += <<-INSTALL - fetchGitFormula #{formula[:source]} "#{formula[:name]}" "#{formula[:branch] || 'master'}" "#{config[:root_path]}/#{config[:ssh_home]}/#{File.basename(formula[:sshkey])}" + fetchGitFormula #{formula[:source]} "#{formula[:name]}" "#{formula[:branch] || 'master'}" "#{ssh_key}" INSTALL end when 'spm' diff --git a/lib/kitchen/provisioner/formula-fetch.sh b/lib/kitchen/provisioner/formula-fetch.sh index 60d5abb5..0a54d58a 100755 --- a/lib/kitchen/provisioner/formula-fetch.sh +++ b/lib/kitchen/provisioner/formula-fetch.sh @@ -34,11 +34,14 @@ function fetchDependencies() { function fetchGitFormula() { test -n "${FETCHED}" || declare -a FETCHED=() export GIT_FORMULAS_PATH=${GIT_FORMULAS_PATH:-/usr/share/salt-formulas/env/_formulas} - if [ $4 != "NULL" ] + + if [[ -n $4 ]] then sshbin=$(which ssh) export GIT_SSH_COMMAND="${sshbin} -o UserKnownHostsFile=/tmp/kitchen/ssh/known_hosts -o StrictHostKeyChecking=no -i ${4}" + export GIT_SSH="/tmp/kitchen/git_ssh.sh" fi + mkdir -p "$GIT_FORMULAS_PATH" if [ -n "$1" ]; then source="$1" @@ -54,7 +57,7 @@ function fetchGitFormula() { popd &>/dev/null || exit else echo "git clone $source $GIT_FORMULAS_PATH/$name -b $branch" - git clone "$source" "$GIT_FORMULAS_PATH/$name" -b "$branch" + git clone "$source" "$GIT_FORMULAS_PATH/$name" -b "$branch" || exit 1 fi # install dependencies FETCHED+=("$name") diff --git a/lib/kitchen/provisioner/git_ssh.sh b/lib/kitchen/provisioner/git_ssh.sh new file mode 100755 index 00000000..c2172703 --- /dev/null +++ b/lib/kitchen/provisioner/git_ssh.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# Workaround: GIT_SSH_COMMAND isn't supported by Git < 2.3 +exec ${GIT_SSH_COMMAND:-ssh} "$@" diff --git a/lib/kitchen/provisioner/salt_solo.rb b/lib/kitchen/provisioner/salt_solo.rb index af0bb9ad..7b59b308 100644 --- a/lib/kitchen/provisioner/salt_solo.rb +++ b/lib/kitchen/provisioner/salt_solo.rb @@ -81,7 +81,8 @@ class SaltSolo < Base salt_yum_repo_latest: 'https://repo.saltstack.com/yum/redhat/salt-repo-latest-2.el7.noarch.rpm', salt_yum_repo: 'https://repo.saltstack.com/yum/redhat/$releasever/$basearch/archive/%s', salt_yum_rpm_key: 'https://repo.saltstack.com/yum/redhat/7/x86_64/archive/%s/SALTSTACK-GPG-KEY.pub', - ssh_home: '/ssh/', + ssh_home: '/ssh', + ssh_key: nil, state_collection: false, state_top_from_file: false, state_top: {}, @@ -434,12 +435,25 @@ def prepare_dependencies ======= # Write ssh known_hosts write_raw_file(File.join(sandbox_path, config[:ssh_home], "known_hosts"), File.read(File.expand_path("../known_hosts", __FILE__))) - # Write git deploy keys. + # Write general deploy key. + unless config[:ssh_key].nil? + outfile = File.join(sandbox_path, config[:ssh_home], File.basename(config[:ssh_key])) + contents = File.read(File.expand_path(config[:ssh_key])) + if contents.include?("ENCRYPTED") + raise("Encrypted key not supported offending key: #{config[:ssh_key]}") + end + info("Copying #{config[:ssh_key]} to #{outfile}") + write_raw_file(outfile, contents) + end + # Write dependency overridden deploykey config[:dependencies].each do |dependency| - if dependency[:sshkey].present? - outfile = File.join(sandbox_path, config[:ssh_home], File.basename(dependency[:sshkey])) - contents = File.read(File.expand_path(dependency[:sshkey])) - info("Copying #{dependency[:sshkey]} to #{outfile}") + unless dependency[:ssh_key].nil? + outfile = File.join(sandbox_path, config[:ssh_home], File.basename(dependency[:ssh_key])) + contents = File.read(File.expand_path(dependency[:ssh_key])) + if contents.include?("ENCRYPTED") + raise("Encrypted key not supported offending key: #{dependency[:ssh_key]}") + end + info("Copying #{dependency[:ssh_key]} to #{outfile}") write_raw_file(outfile, contents) end end @@ -470,7 +484,7 @@ def prepare_dependencies end # upload scripts - %w[formula-fetch.sh repository-setup.sh].each do |script| + %w[formula-fetch.sh repository-setup.sh git_ssh.sh].each do |script| write_raw_file(File.join(sandbox_path, script), File.read(File.expand_path("../#{script}", __FILE__))) end dependencies_script = File.expand_path('./../dependencies.erb', __FILE__) From 5f9dddbc060380bd38f4029ec9e7e2a7b46a8e23 Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Wed, 26 Feb 2020 08:33:08 +0100 Subject: [PATCH 4/9] fix: SC2086: Double quote to prevent globbing and word splitting. --- lib/kitchen/provisioner/git_ssh.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kitchen/provisioner/git_ssh.sh b/lib/kitchen/provisioner/git_ssh.sh index c2172703..c69e8edc 100755 --- a/lib/kitchen/provisioner/git_ssh.sh +++ b/lib/kitchen/provisioner/git_ssh.sh @@ -1,3 +1,3 @@ #!/bin/sh # Workaround: GIT_SSH_COMMAND isn't supported by Git < 2.3 -exec ${GIT_SSH_COMMAND:-ssh} "$@" +exec "${GIT_SSH_COMMAND:-ssh}" "$@" From 81d3fc8954497265d6cceb8b5c628ad440d586ba Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Wed, 26 Feb 2020 09:02:39 +0100 Subject: [PATCH 5/9] Commit signed with pgp key --- lib/kitchen/provisioner/git_ssh.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kitchen/provisioner/git_ssh.sh b/lib/kitchen/provisioner/git_ssh.sh index c69e8edc..c705d8ea 100755 --- a/lib/kitchen/provisioner/git_ssh.sh +++ b/lib/kitchen/provisioner/git_ssh.sh @@ -1,3 +1,3 @@ #!/bin/sh -# Workaround: GIT_SSH_COMMAND isn't supported by Git < 2.3 +# Workaround: GIT_SSH_COMMAND is not supported by Git < 2.3 exec "${GIT_SSH_COMMAND:-ssh}" "$@" From bc7c1437d609338badbb0f4c146f0b368a6a1b8f Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Thu, 16 Apr 2020 19:08:41 +0200 Subject: [PATCH 6/9] force new build --- lib/kitchen/provisioner/dependencies.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/kitchen/provisioner/dependencies.erb b/lib/kitchen/provisioner/dependencies.erb index babacd99..f6f9c4ee 100644 --- a/lib/kitchen/provisioner/dependencies.erb +++ b/lib/kitchen/provisioner/dependencies.erb @@ -1,5 +1,6 @@ <%= + def install_dependencies script = '' From 43bb74878c8449f7031222ed6dc205dc9231c727 Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Thu, 16 Apr 2020 19:08:58 +0200 Subject: [PATCH 7/9] force new build --- lib/kitchen/provisioner/dependencies.erb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/kitchen/provisioner/dependencies.erb b/lib/kitchen/provisioner/dependencies.erb index f6f9c4ee..babacd99 100644 --- a/lib/kitchen/provisioner/dependencies.erb +++ b/lib/kitchen/provisioner/dependencies.erb @@ -1,6 +1,5 @@ <%= - def install_dependencies script = '' From 002223c3f803a317fa475db956e2bf1dea32fabb Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Tue, 13 Apr 2021 13:29:28 +0200 Subject: [PATCH 8/9] fix: SC2230 --- lib/kitchen/provisioner/formula-fetch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kitchen/provisioner/formula-fetch.sh b/lib/kitchen/provisioner/formula-fetch.sh index 0a54d58a..64e1c6af 100755 --- a/lib/kitchen/provisioner/formula-fetch.sh +++ b/lib/kitchen/provisioner/formula-fetch.sh @@ -37,7 +37,7 @@ function fetchGitFormula() { if [[ -n $4 ]] then - sshbin=$(which ssh) + sshbin=$(command -v ssh) export GIT_SSH_COMMAND="${sshbin} -o UserKnownHostsFile=/tmp/kitchen/ssh/known_hosts -o StrictHostKeyChecking=no -i ${4}" export GIT_SSH="/tmp/kitchen/git_ssh.sh" fi From b804bea2e69f056ee12bf2194fd3ed3645808c58 Mon Sep 17 00:00:00 2001 From: Justin Zandbergen Date: Tue, 13 Apr 2021 14:01:16 +0200 Subject: [PATCH 9/9] fix: Missed merge conflict. --- lib/kitchen/provisioner/salt_solo.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/kitchen/provisioner/salt_solo.rb b/lib/kitchen/provisioner/salt_solo.rb index 7b59b308..7284c6c8 100644 --- a/lib/kitchen/provisioner/salt_solo.rb +++ b/lib/kitchen/provisioner/salt_solo.rb @@ -424,7 +424,6 @@ def prepare_grains end def prepare_dependencies -<<<<<<< HEAD # Dependency scripts are bash scripts only # Copying them clobbers the kitchen temp directory # with a file named `kitchen`. If adding Windows @@ -432,7 +431,6 @@ def prepare_dependencies # sub-directory return if windows_os? -======= # Write ssh known_hosts write_raw_file(File.join(sandbox_path, config[:ssh_home], "known_hosts"), File.read(File.expand_path("../known_hosts", __FILE__))) # Write general deploy key. @@ -457,7 +455,7 @@ def prepare_dependencies write_raw_file(outfile, contents) end end ->>>>>>> Add sshkey option for git based formula dependencies. + # upload scripts sandbox_scripts_path = File.join(sandbox_path, config[:salt_config], 'scripts') info("Preparing scripts into #{config[:salt_config]}/scripts")