Detect systemd-resolved stub resolver at localhost and bypass using k…

…ubelet --resolv-conf (#450)

lib/pharos/autoload.rb

@@ -45,6 +45,7 @@ module Terraform
   module Configuration
     autoload :Host, 'pharos/configuration/host'
     autoload :Taint, 'pharos/configuration/taint'
+    autoload :OsRelease, 'pharos/configuration/os_release'
   end
 
   module Etcd

lib/pharos/configuration/host.rb

@@ -8,6 +8,11 @@ module Configuration
     class Host < Dry::Struct
       constructor_type :schema
 
+      class ResolvConf < Dry::Struct
+        attribute :nameserver_localhost, Pharos::Types::Strict::Bool
+        attribute :systemd_resolved_stub, Pharos::Types::Strict::Bool
+      end
+
       attribute :address, Pharos::Types::Strict::String
       attribute :private_address, Pharos::Types::Strict::String
       attribute :private_interface, Pharos::Types::Strict::String
@@ -19,7 +24,7 @@ class Host < Dry::Struct
       attribute :container_runtime, Pharos::Types::Strict::String.default('docker')
       attribute :http_proxy, Pharos::Types::Strict::String
 
-      attr_accessor :os_release, :cpu_arch, :hostname, :api_endpoint, :private_interface_address, :checks
+      attr_accessor :os_release, :cpu_arch, :hostname, :api_endpoint, :private_interface_address, :checks, :resolvconf
 
       def to_s
         address

lib/pharos/phases/configure_kubelet.rb

@@ -97,10 +97,19 @@ def build_systemd_dropin
 
       # @return [Array<String>]
       def kubelet_dns_args
-        [
+        args = [
           "--cluster-dns=#{@config.network.dns_service_ip}",
           "--cluster-domain=cluster.local"
         ]
+
+        if @host.resolvconf.systemd_resolved_stub
+          # use usptream resolvers instead of systemd stub resolver at localhost for `dnsPolicy: Default` pods
+          args << '--resolv-conf=/run/systemd/resolve/resolv.conf'
+        elsif @host.resolvconf.nameserver_localhost
+          fail "Host has /etc/resolv.conf configured with localhost as a resolver"
+        end
+
+        args
       end
 
       # @return [Array<String>]

lib/pharos/phases/validate_host.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'ipaddr'
+
 module Pharos
   module Phases
     class ValidateHost < Pharos::Phase
@@ -42,6 +44,7 @@ def gather_host_facts
         @host.hostname = hostname
         @host.checks = host_checks
         @host.private_interface_address = private_interface_address(@host.private_interface) if @host.private_interface
+        @host.resolvconf = read_resolvconf
       end
 
       def check_role
@@ -123,6 +126,43 @@ def private_interface_address(interface)
         end
         nil
       end
+
+      # @return [Array<String>]
+      def read_resolvconf_nameservers
+        nameservers = []
+
+        @ssh.file('/etc/resolv.conf').each_line do |line|
+          if match = line.match(/nameserver (.+)/)
+            nameservers << match[1]
+          end
+        end
+
+        nameservers
+      end
+
+      LOCALNET = IPAddr.new('127.0.0.0/8')
+
+      # Host /etc/resolv.conf is configured to use a nameserver at localhost in the host network namespace
+      # @return [Boolean]
+      def check_resolvconf_nameserver_localhost
+        resolvers = read_resolvconf_nameservers.map{ |ip| IPAddr.new(ip) }
+        resolvers.any? { |ip| LOCALNET.include?(ip) }
+      end
+
+      # Host /etc/resolv.conf is configured to use the systemd-resolved stub resolver at 127.0.0.53
+      # @return [Boolean]
+      def check_resolvconf_systemd_resolved_stub
+        symlink = @ssh.file('/etc/resolv.conf').readlink
+        !!symlink && symlink.end_with?('/run/systemd/resolve/stub-resolv.conf')
+      end
+
+      # @return [Pharos::Configuration::Host::ResolvConf]
+      def read_resolvconf
+        Pharos::Configuration::Host::ResolvConf.new(
+          nameserver_localhost: check_resolvconf_nameserver_localhost,
+          systemd_resolved_stub: check_resolvconf_systemd_resolved_stub
+        )
+      end
     end
   end
 end

lib/pharos/ssh/remote_file.rb

@@ -82,6 +82,15 @@ def link(target)
         @client.exec!("sudo ln -s #{escaped_path} #{target.shellescape}")
       end
 
+      # @return [String, nil]
+      def readlink
+        target = @client.exec!("readlink #{escaped_path} || echo").strip
+
+        return nil if target.empty?
+
+        target
+      end
+
       # Yields each line in the remote file
       # @yield [String]
       def each_line

spec/pharos/phases/configure_kubelet_spec.rb

@@ -1,11 +1,19 @@
 require "pharos/phases/configure_kubelet"
 
 describe Pharos::Phases::ConfigureKubelet do
-  let(:host) { Pharos::Configuration::Host.new(address: 'test', private_address: '192.168.42.1') }
+  let(:host_resolvconf) { Pharos::Configuration::Host::ResolvConf.new(
+      nameserver_localhost: false,
+      systemd_resolved_stub: false,
+  ) }
+  let(:host) { Pharos::Configuration::Host.new(
+    address: 'test',
+    private_address: '192.168.42.1',
+  ) }
 
+  let(:config_network) { { }}
   let(:config) { Pharos::Config.new(
       hosts: [host],
-      network: {},
+      network: config_network,
       addons: {},
       etcd: {},
       kubelet: {read_only_port: false}
@@ -15,6 +23,8 @@
   subject { described_class.new(host, config: config, ssh: ssh) }
 
   before(:each) do
+    host.resolvconf = host_resolvconf
+
     allow(host).to receive(:cpu_arch).and_return(double(:cpu_arch, name: 'amd64'))
   end
 
@@ -27,22 +37,6 @@
         ExecStartPre=-/sbin/swapoff -a
       EOM
     end
-
-    context "with a different network.service_cidr" do
-      let(:config) { Pharos::Config.new(
-          hosts: [host],
-          network: {
-            service_cidr: '172.255.0.0/16',
-          },
-          addons: {},
-          etcd: {},
-          kubelet: {}
-      ) }
-
-      it "uses the customized --cluster-dns" do
-        expect(subject.build_systemd_dropin).to match /KUBELET_DNS_ARGS=--cluster-dns=172.255.0.10 --cluster-domain=cluster.local/
-      end
-    end
   end
 
   describe "#kubelet_extra_args" do
@@ -110,4 +104,52 @@
       end
     end
   end
+
+  describe '#kubelet_dns_args' do
+    it 'returns cluster service IP' do
+      expect(subject.kubelet_dns_args).to eq [
+        '--cluster-dns=10.96.0.10',
+        '--cluster-domain=cluster.local',
+      ]
+    end
+
+    context "with a different network.service_cidr" do
+      let(:config_network) { {
+          service_cidr: '172.255.0.0/16',
+      } }
+
+      it "uses the customized --cluster-dns" do
+        expect(subject.kubelet_dns_args).to eq [
+          '--cluster-dns=172.255.0.10',
+          '--cluster-domain=cluster.local',
+        ]
+      end
+    end
+
+    context "with a systemd-resolved stub" do
+      let(:host_resolvconf) { Pharos::Configuration::Host::ResolvConf.new(
+          nameserver_localhost: true,
+          systemd_resolved_stub: true,
+      ) }
+
+      it "uses --resolv-conf" do
+        expect(subject.kubelet_dns_args).to eq [
+          '--cluster-dns=10.96.0.10',
+          '--cluster-domain=cluster.local',
+          '--resolv-conf=/run/systemd/resolve/resolv.conf',
+        ]
+      end
+    end
+
+    context "with a non-systemd-resolved localhost resolver" do
+      let(:host_resolvconf) { Pharos::Configuration::Host::ResolvConf.new(
+          nameserver_localhost: true,
+          systemd_resolved_stub: false,
+      ) }
+
+      it "fails" do
+        expect{subject.kubelet_dns_args}.to raise_error 'Host has /etc/resolv.conf configured with localhost as a resolver'
+      end
+    end
+  end
 end

spec/pharos/phases/validate_host_spec.rb

@@ -148,4 +148,66 @@
       end
     end
   end
+
+  describe '#get_resolvconf' do
+    let(:file) { instance_double(Pharos::SSH::RemoteFile) }
+    let(:file_readlink) { nil }
+
+    before do
+      allow(ssh).to receive(:file).with('/etc/resolv.conf').and_return(file)
+
+      mock = allow(file).to receive(:each_line)
+      file_lines.each do |line|
+        mock = mock.and_yield(line)
+      end
+
+      allow(file).to receive(:readlink).and_return(file_readlink)
+    end
+
+    context 'for a normal resolv.conf' do
+      let(:file_lines) { ['nameserver 8.8.8.8'] }
+
+      it 'returns ok' do
+        expect(subject.read_resolvconf).to eq Pharos::Configuration::Host::ResolvConf.new(
+          nameserver_localhost: false,
+          systemd_resolved_stub: false,
+        )
+      end
+    end
+
+    context 'for a normal resolv.conf with localhost' do
+      let(:file_lines) { ['nameserver 127.0.0.53'] }
+
+      it 'returns nameserver_localhost' do
+        expect(subject.read_resolvconf).to eq Pharos::Configuration::Host::ResolvConf.new(
+          nameserver_localhost: true,
+          systemd_resolved_stub: false,
+        )
+      end
+    end
+
+    context 'for a systemd-resolved resolv.conf stub' do
+      let(:file_lines) { ['nameserver 127.0.0.53'] }
+      let(:file_readlink) { '../run/systemd/resolve/stub-resolv.conf' }
+
+      it 'returns systemd_resolved_stub' do
+        expect(subject.read_resolvconf).to eq Pharos::Configuration::Host::ResolvConf.new(
+          nameserver_localhost: true,
+          systemd_resolved_stub: true,
+        )
+      end
+    end
+
+    context 'for a non-resolved resolv.conf symlink' do
+      let(:file_lines) { ['nameserver 8.8.8.8'] }
+      let(:file_readlink) { '/run/resolvconf/resolv.conf' }
+
+      it 'returns ok' do
+        expect(subject.read_resolvconf).to eq Pharos::Configuration::Host::ResolvConf.new(
+          nameserver_localhost: false,
+          systemd_resolved_stub: false,
+        )
+      end
+    end
+  end
 end