diff --git a/lib/pharos/autoload.rb b/lib/pharos/autoload.rb index 6fcb689bb..1a2aeb4f1 100644 --- a/lib/pharos/autoload.rb +++ b/lib/pharos/autoload.rb @@ -45,6 +45,7 @@ module Terraform module Configuration autoload :Host, 'pharos/configuration/host' autoload :Taint, 'pharos/configuration/taint' + autoload :OsRelease, 'pharos/configuration/os_release' end module Etcd diff --git a/lib/pharos/configuration/host.rb b/lib/pharos/configuration/host.rb index a69b7ee84..369da8d91 100644 --- a/lib/pharos/configuration/host.rb +++ b/lib/pharos/configuration/host.rb @@ -8,6 +8,11 @@ module Configuration class Host < Dry::Struct constructor_type :schema + class ResolvConf < Dry::Struct + attribute :nameserver_localhost, Pharos::Types::Strict::Bool + attribute :systemd_resolved_stub, Pharos::Types::Strict::Bool + end + attribute :address, Pharos::Types::Strict::String attribute :private_address, Pharos::Types::Strict::String attribute :private_interface, Pharos::Types::Strict::String @@ -19,7 +24,7 @@ class Host < Dry::Struct attribute :container_runtime, Pharos::Types::Strict::String.default('docker') attribute :http_proxy, Pharos::Types::Strict::String - attr_accessor :os_release, :cpu_arch, :hostname, :api_endpoint, :private_interface_address, :checks + attr_accessor :os_release, :cpu_arch, :hostname, :api_endpoint, :private_interface_address, :checks, :resolvconf def to_s address diff --git a/lib/pharos/phases/configure_kubelet.rb b/lib/pharos/phases/configure_kubelet.rb index 3134b5076..25d140918 100644 --- a/lib/pharos/phases/configure_kubelet.rb +++ b/lib/pharos/phases/configure_kubelet.rb @@ -97,10 +97,19 @@ def build_systemd_dropin # @return [Array] def kubelet_dns_args - [ + args = [ "--cluster-dns=#{@config.network.dns_service_ip}", "--cluster-domain=cluster.local" ] + + if @host.resolvconf.systemd_resolved_stub + # use usptream resolvers instead of systemd stub resolver at localhost for `dnsPolicy: Default` pods + args << '--resolv-conf=/run/systemd/resolve/resolv.conf' + elsif @host.resolvconf.nameserver_localhost + fail "Host has /etc/resolv.conf configured with localhost as a resolver" + end + + args end # @return [Array] diff --git a/lib/pharos/phases/validate_host.rb b/lib/pharos/phases/validate_host.rb index 8cc9b4af8..5ed3b6a02 100644 --- a/lib/pharos/phases/validate_host.rb +++ b/lib/pharos/phases/validate_host.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'ipaddr' + module Pharos module Phases class ValidateHost < Pharos::Phase @@ -42,6 +44,7 @@ def gather_host_facts @host.hostname = hostname @host.checks = host_checks @host.private_interface_address = private_interface_address(@host.private_interface) if @host.private_interface + @host.resolvconf = read_resolvconf end def check_role @@ -123,6 +126,43 @@ def private_interface_address(interface) end nil end + + # @return [Array] + def read_resolvconf_nameservers + nameservers = [] + + @ssh.file('/etc/resolv.conf').each_line do |line| + if match = line.match(/nameserver (.+)/) + nameservers << match[1] + end + end + + nameservers + end + + LOCALNET = IPAddr.new('127.0.0.0/8') + + # Host /etc/resolv.conf is configured to use a nameserver at localhost in the host network namespace + # @return [Boolean] + def check_resolvconf_nameserver_localhost + resolvers = read_resolvconf_nameservers.map{ |ip| IPAddr.new(ip) } + resolvers.any? { |ip| LOCALNET.include?(ip) } + end + + # Host /etc/resolv.conf is configured to use the systemd-resolved stub resolver at 127.0.0.53 + # @return [Boolean] + def check_resolvconf_systemd_resolved_stub + symlink = @ssh.file('/etc/resolv.conf').readlink + !!symlink && symlink.end_with?('/run/systemd/resolve/stub-resolv.conf') + end + + # @return [Pharos::Configuration::Host::ResolvConf] + def read_resolvconf + Pharos::Configuration::Host::ResolvConf.new( + nameserver_localhost: check_resolvconf_nameserver_localhost, + systemd_resolved_stub: check_resolvconf_systemd_resolved_stub + ) + end end end end diff --git a/lib/pharos/ssh/remote_file.rb b/lib/pharos/ssh/remote_file.rb index b80ce9d4a..f5cf88b7e 100644 --- a/lib/pharos/ssh/remote_file.rb +++ b/lib/pharos/ssh/remote_file.rb @@ -82,6 +82,15 @@ def link(target) @client.exec!("sudo ln -s #{escaped_path} #{target.shellescape}") end + # @return [String, nil] + def readlink + target = @client.exec!("readlink #{escaped_path} || echo").strip + + return nil if target.empty? + + target + end + # Yields each line in the remote file # @yield [String] def each_line diff --git a/spec/pharos/phases/configure_kubelet_spec.rb b/spec/pharos/phases/configure_kubelet_spec.rb index e56cca1c7..98ccd0d9b 100644 --- a/spec/pharos/phases/configure_kubelet_spec.rb +++ b/spec/pharos/phases/configure_kubelet_spec.rb @@ -1,11 +1,19 @@ require "pharos/phases/configure_kubelet" describe Pharos::Phases::ConfigureKubelet do - let(:host) { Pharos::Configuration::Host.new(address: 'test', private_address: '192.168.42.1') } + let(:host_resolvconf) { Pharos::Configuration::Host::ResolvConf.new( + nameserver_localhost: false, + systemd_resolved_stub: false, + ) } + let(:host) { Pharos::Configuration::Host.new( + address: 'test', + private_address: '192.168.42.1', + ) } + let(:config_network) { { }} let(:config) { Pharos::Config.new( hosts: [host], - network: {}, + network: config_network, addons: {}, etcd: {}, kubelet: {read_only_port: false} @@ -15,6 +23,8 @@ subject { described_class.new(host, config: config, ssh: ssh) } before(:each) do + host.resolvconf = host_resolvconf + allow(host).to receive(:cpu_arch).and_return(double(:cpu_arch, name: 'amd64')) end @@ -27,22 +37,6 @@ ExecStartPre=-/sbin/swapoff -a EOM end - - context "with a different network.service_cidr" do - let(:config) { Pharos::Config.new( - hosts: [host], - network: { - service_cidr: '172.255.0.0/16', - }, - addons: {}, - etcd: {}, - kubelet: {} - ) } - - it "uses the customized --cluster-dns" do - expect(subject.build_systemd_dropin).to match /KUBELET_DNS_ARGS=--cluster-dns=172.255.0.10 --cluster-domain=cluster.local/ - end - end end describe "#kubelet_extra_args" do @@ -110,4 +104,52 @@ end end end + + describe '#kubelet_dns_args' do + it 'returns cluster service IP' do + expect(subject.kubelet_dns_args).to eq [ + '--cluster-dns=10.96.0.10', + '--cluster-domain=cluster.local', + ] + end + + context "with a different network.service_cidr" do + let(:config_network) { { + service_cidr: '172.255.0.0/16', + } } + + it "uses the customized --cluster-dns" do + expect(subject.kubelet_dns_args).to eq [ + '--cluster-dns=172.255.0.10', + '--cluster-domain=cluster.local', + ] + end + end + + context "with a systemd-resolved stub" do + let(:host_resolvconf) { Pharos::Configuration::Host::ResolvConf.new( + nameserver_localhost: true, + systemd_resolved_stub: true, + ) } + + it "uses --resolv-conf" do + expect(subject.kubelet_dns_args).to eq [ + '--cluster-dns=10.96.0.10', + '--cluster-domain=cluster.local', + '--resolv-conf=/run/systemd/resolve/resolv.conf', + ] + end + end + + context "with a non-systemd-resolved localhost resolver" do + let(:host_resolvconf) { Pharos::Configuration::Host::ResolvConf.new( + nameserver_localhost: true, + systemd_resolved_stub: false, + ) } + + it "fails" do + expect{subject.kubelet_dns_args}.to raise_error 'Host has /etc/resolv.conf configured with localhost as a resolver' + end + end + end end diff --git a/spec/pharos/phases/validate_host_spec.rb b/spec/pharos/phases/validate_host_spec.rb index eaf0d36ba..9c9097aee 100644 --- a/spec/pharos/phases/validate_host_spec.rb +++ b/spec/pharos/phases/validate_host_spec.rb @@ -148,4 +148,66 @@ end end end + + describe '#get_resolvconf' do + let(:file) { instance_double(Pharos::SSH::RemoteFile) } + let(:file_readlink) { nil } + + before do + allow(ssh).to receive(:file).with('/etc/resolv.conf').and_return(file) + + mock = allow(file).to receive(:each_line) + file_lines.each do |line| + mock = mock.and_yield(line) + end + + allow(file).to receive(:readlink).and_return(file_readlink) + end + + context 'for a normal resolv.conf' do + let(:file_lines) { ['nameserver 8.8.8.8'] } + + it 'returns ok' do + expect(subject.read_resolvconf).to eq Pharos::Configuration::Host::ResolvConf.new( + nameserver_localhost: false, + systemd_resolved_stub: false, + ) + end + end + + context 'for a normal resolv.conf with localhost' do + let(:file_lines) { ['nameserver 127.0.0.53'] } + + it 'returns nameserver_localhost' do + expect(subject.read_resolvconf).to eq Pharos::Configuration::Host::ResolvConf.new( + nameserver_localhost: true, + systemd_resolved_stub: false, + ) + end + end + + context 'for a systemd-resolved resolv.conf stub' do + let(:file_lines) { ['nameserver 127.0.0.53'] } + let(:file_readlink) { '../run/systemd/resolve/stub-resolv.conf' } + + it 'returns systemd_resolved_stub' do + expect(subject.read_resolvconf).to eq Pharos::Configuration::Host::ResolvConf.new( + nameserver_localhost: true, + systemd_resolved_stub: true, + ) + end + end + + context 'for a non-resolved resolv.conf symlink' do + let(:file_lines) { ['nameserver 8.8.8.8'] } + let(:file_readlink) { '/run/resolvconf/resolv.conf' } + + it 'returns ok' do + expect(subject.read_resolvconf).to eq Pharos::Configuration::Host::ResolvConf.new( + nameserver_localhost: false, + systemd_resolved_stub: false, + ) + end + end + end end