Merge branch 'master' into release/v3.1.0

.travis.yml

@@ -16,18 +16,11 @@ jobs:
       name: "e2e: xenial docker with weave"
       script: ./e2e/travis.sh
       rvm: 2.5.4
-      dist: xenial
+      dist: bionic
       env:
         - CONTAINER_RUNTIME=docker
         - NETWORK_PROVIDER=weave
-    - stage: e2e
-      name: "e2e: xenial docker with calico"
-      script: ./e2e/travis.sh
-      rvm: 2.5.4
-      dist: xenial
-      env:
-        - CONTAINER_RUNTIME=docker
-        - NETWORK_PROVIDER=calico
+        - FOOTLOOSE_IMAGE=quay.io/footloose/ubuntu16.04
     - stage: e2e
       name: "e2e: bionic docker with weave"
       script: ./e2e/travis.sh
@@ -36,14 +29,16 @@ jobs:
       env:
         - CONTAINER_RUNTIME=docker
         - NETWORK_PROVIDER=weave
+        - FOOTLOOSE_IMAGE=quay.io/footloose/ubuntu18.04
     - stage: e2e
-      name: "e2e: bionic bdocker with calico"
+      name: "e2e: centos7 docker with weave"
       script: ./e2e/travis.sh
       rvm: 2.5.4
       dist: bionic
       env:
         - CONTAINER_RUNTIME=docker
-        - NETWORK_PROVIDER=calico
+        - NETWORK_PROVIDER=weave
+        - FOOTLOOSE_IMAGE=quay.io/footloose/centos7
     - stage: publish binary
       script: ./build/travis/macos.sh
       os: osx

e2e/cluster.yml

@@ -1,10 +1,22 @@
 hosts:
   - address: 127.0.0.1
-    private_interface: ens4
-    user: travis
+    private_interface: eth0
+    ssh_port: 9022
+    user: root
     role: master
     ssh_key_path: ~/.ssh/id_rsa_travis
     container_runtime: $CONTAINER_RUNTIME
-    taints: []
+  - address: 127.0.0.1
+    private_interface: eth0
+    ssh_port: 9023
+    user: root
+    role: worker
+    ssh_key_path: ~/.ssh/id_rsa_travis
+    container_runtime: $CONTAINER_RUNTIME
 network:
   provider: $NETWORK_PROVIDER
+  pod_network_cidr: 172.20.0.0/16
+  service_cidr: 172.19.0.0/16
+kube_proxy:
+  conntrack:
+    maxPerCore: 0

e2e/footloose.yaml

@@ -0,0 +1,38 @@
+cluster:
+  name: pharos
+  privateKey: ~/.ssh/id_rsa_travis
+machines:
+- count: 1
+  backend: docker
+  spec:
+    image: $FOOTLOOSE_IMAGE # quay.io/footloose/ubuntu18.04
+    name: master%d
+    privileged: true
+    volumes:
+    - type: volume
+      destination: /var/lib/docker
+    - type: volume
+      destination: /var/lib/kubelet
+    - type: volume
+      destination: /var/lib/containerd
+    portMappings:
+    - containerPort: 22
+      hostPort: 9022
+    - containerPort: 6443
+      hostPort: 6443
+- count: 1
+  backend: docker
+  spec:
+    image: $FOOTLOOSE_IMAGE # quay.io/footloose/ubuntu18.04
+    name: worker%d
+    privileged: true
+    volumes:
+    - type: volume
+      destination: /var/lib/docker
+    - type: volume
+      destination: /var/lib/kubelet
+    - type: volume
+      destination: /var/lib/containerd
+    portMappings:
+    - containerPort: 22
+      hostPort: 9022

e2e/travis.sh

@@ -5,13 +5,8 @@ set -ue
 
 source ./e2e/util.sh
 
-if [ "${CONTAINER_RUNTIME}" != "docker" ]; then
-    echo "Stopping docker ..."
-    sudo systemctl stop docker
-    sudo systemctl disable docker
-    sudo apt-get remove --purge docker-ce
-    sudo rm -f /var/run/docker.sock
-fi
+curl -fsSL https://get.docker.com -o get-docker.sh
+sudo sh get-docker.sh
 
 ssh-keygen -t rsa -f ~/.ssh/id_rsa_travis -N ""
 cat ~/.ssh/id_rsa_travis.pub > ~/.ssh/authorized_keys
@@ -20,29 +15,45 @@ chmod 0600 ~/.ssh/authorized_keys
 ifconfig
 
 envsubst < e2e/cluster.yml > cluster.yml
+envsubst < e2e/footloose.yaml > footloose.yaml
+
+curl -L https://github.com/weaveworks/footloose/releases/download/0.6.3/footloose-0.6.3-linux-x86_64 > ./footloose
+chmod +x ./footloose
+./footloose create
+./footloose ssh root@master0 -- 'apt-get install -y curl || yum install -y curl which openssh-clients'
+./footloose ssh root@worker0 -- 'apt-get install -y curl || yum install -y curl which openssh-clients'
 
 bundle exec bin/pharos
 bundle exec bin/pharos -v
 bundle exec bin/pharos version
 bundle exec bin/pharos up -y -c cluster.yml
 bundle exec bin/pharos ssh --role master -c cluster.yml -- kubectl get nodes
 bundle exec bin/pharos kubeconfig -c cluster.yml > kubeconfig.e2e
+export KUBECONFIG=./kubeconfig.e2e
 
 # Verify that workloads start running
 curl -sLO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
 chmod +x ./kubectl
 sudo mv kubectl /usr/local/bin/
-export KUBECONFIG=./kubeconfig.e2e
 
-echo "==> Check that metrics-server is running:"
+echo "==> Checking that metrics-server is running:"
 (retry 30 pods_running "k8s-app=metrics-server" "kube-system") || exit $?
 
 echo "==> Test with sonobuoy"
 curl -L https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.18.0/sonobuoy_0.18.0_linux_amd64.tar.gz | tar xzv
 chmod +x ./sonobuoy
-./sonobuoy run --mode quick --timeout 600 --wait
-
-echo "==> Test re-up"
+(
+  sleep 30
+  ./sonobuoy logs -f
+)&
+logs_pid=$!
+./sonobuoy run --wait=600 --plugin-env=e2e.E2E_USE_GO_RUNNER=true '--e2e-focus=\[sig-network\].*\[Conformance\]' '--e2e-skip=\[Serial\]' --e2e-parallel=y
+kill $logs_pid
+results=$(./sonobuoy retrieve)
+./sonobuoy results "${results}"
+./sonobuoy status | grep -q -E ' +e2e +complete +passed +'
+
+# Test re-up
 bundle exec bin/pharos up -y -c cluster.yml
 
 echo "==> Test reset"

lib/pharos/config_schema.rb

@@ -11,7 +11,10 @@ class ConfigSchema
       'network' => {},
       'authentication' => {},
       'kube_proxy' => {},
-      'kubelet' => {},
+      'kubelet' => {
+        'system_reserved' => { 'cpu' => '50m', 'memory' => '50Mi' },
+        'kube_reserved' => { 'cpu' => '100m', 'memory' => '200Mi' }
+      },
       'telemetry' => {},
       'pod_security_policy' => {},
       'addon_paths' => [],
@@ -198,13 +201,16 @@ def unique_addresses?(hosts)
         end
         optional(:kube_proxy).schema do
           optional(:mode).filled(included_in?: %w(userspace iptables ipvs))
+          optional(:conntrack).filled
         end
         optional(:kubelet).schema do
           optional(:read_only_port).filled(:bool?)
           optional(:feature_gates).filled
           optional(:extra_args).each(type?: String)
           optional(:cpu_cfs_quota).filled(:bool?)
           optional(:cpu_cfs_quota_period).filled(:str?)
+          optional(:system_reserved).filled
+          optional(:kube_reserved).filled
         end
         optional(:control_plane).schema do
           optional(:use_proxy).filled(:bool?)

lib/pharos/configuration/host.rb

@@ -104,6 +104,7 @@ def kubelet_args(local_only: false, cloud_provider: nil)
         args = config&.kubelet&.extra_args.dup || []
 
         args << "--rotate-server-certificates"
+        args << "--fail-swap-on=false"
 
         if local_only
           args << "--pod-manifest-path=/etc/kubernetes/manifests/"

lib/pharos/configuration/kube_proxy.rb

@@ -4,6 +4,7 @@ module Pharos
   module Configuration
     class KubeProxy < Pharos::Configuration::Struct
       attribute :mode, Pharos::Types::String.default('iptables')
+      attribute :conntrack, Pharos::Types::Strict::Hash
     end
   end
 end

lib/pharos/configuration/kubelet.rb

@@ -8,6 +8,8 @@ class Kubelet < Pharos::Configuration::Struct
       attribute :extra_args, Pharos::Types::Strict::Array.of(Pharos::Types::String)
       attribute :cpu_cfs_quota, Pharos::Types::Bool.default(true)
       attribute :cpu_cfs_quota_period, Pharos::Types::String
+      attribute :system_reserved, Pharos::Types::Strict::Hash
+      attribute :kube_reserved, Pharos::Types::Strict::Hash
     end
   end
 end

lib/pharos/host/debian/scripts/configure-netfilter.sh

@@ -2,6 +2,10 @@
 
 set -e
 
+if grep "container=docker" /proc/1/environ ; then
+  exit 0
+fi
+
 /sbin/modprobe br_netfilter
 echo "br_netfilter" > /etc/modules-load.d/br_netfilter.conf
 echo "net.bridge.bridge-nf-call-iptables = 1" > /etc/sysctl.d/99-net-bridge.conf

lib/pharos/host/el7/scripts/configure-docker.sh

@@ -26,6 +26,11 @@ EOF
 
 yum_install_with_lock "docker-ce" "${DOCKER_VERSION}"
 
+if ! systemctl is-active --quiet containerd; then
+    systemctl enable containerd
+    systemctl start containerd
+fi
+
 if ! systemctl is-active --quiet docker; then
     systemctl enable docker
     systemctl start docker

lib/pharos/host/el7/scripts/configure-essentials.sh

@@ -62,6 +62,10 @@ if [[ $PATH != *local/bin* ]] || [[ $PATH != *usr/sbin* ]]; then
   lineinfile "^PATH=" "PATH=$PATH" "$env_file"
 fi
 
+if grep "container=docker" /proc/1/environ ; then
+  exit 0
+fi
+
 if ! (getenforce | grep -q "Disabled"); then
     setenforce 0 || true
     lineinfile "^SELINUX=" "SELINUX=permissive" "/etc/selinux/config"

lib/pharos/host/el7/scripts/configure-netfilter.sh

@@ -2,6 +2,10 @@
 
 set -e
 
+if grep "container=docker" /proc/1/environ ; then
+  exit 0
+fi
+
 /sbin/modprobe br_netfilter
 echo "br_netfilter" > /etc/modules-load.d/br_netfilter.conf
 echo "net.bridge.bridge-nf-call-iptables = 1" > /etc/sysctl.d/99-net-bridge.conf

lib/pharos/host/ubuntu/scripts/configure-docker.sh

@@ -37,3 +37,13 @@ else
     apt-get install -y "$DOCKER_PACKAGE=5:$DOCKER_VERSION*"
 fi
 apt-mark hold "$DOCKER_PACKAGE"
+
+if ! systemctl is-active --quiet containerd; then
+    systemctl enable containerd
+    systemctl start containerd
+fi
+
+if ! systemctl is-active --quiet docker; then
+    systemctl enable docker
+    systemctl start docker
+fi

lib/pharos/host/ubuntu/scripts/configure-netfilter.sh

@@ -2,6 +2,10 @@
 
 set -e
 
+if grep "container=docker" /proc/1/environ ; then
+  exit 0
+fi
+
 modprobe br_netfilter
 echo "br_netfilter" > /etc/modules-load.d/br_netfilter.conf
 echo "net.bridge.bridge-nf-call-iptables = 1" > /etc/sysctl.d/99-net-bridge.conf

lib/pharos/host/ubuntu/scripts/ensure-kubelet.sh

@@ -19,3 +19,8 @@ export DEBIAN_FRONTEND=noninteractive
 apt-mark unhold kubelet kubernetes-cni || echo "Nothing to unhold"
 apt-get install -y "kubelet=${KUBE_VERSION}-00" "kubernetes-cni=${CNI_VERSION}-00"
 apt-mark hold kubelet kubernetes-cni
+
+if ! systemctl is-active --quiet kubelet; then
+    systemctl enable kubelet
+    systemctl start kubelet
+fi

lib/pharos/kubeadm/kubelet_config.rb

@@ -30,6 +30,12 @@ def generate
         if @config.kubelet&.read_only_port
           config['readOnlyPort'] = 10_255
         end
+        if @config.kubelet&.system_reserved
+          config['systemReserved'] = @config.kubelet.system_reserved
+        end
+        if @config.kubelet&.kube_reserved
+          config['kubeReserved'] = @config.kubelet.kube_reserved
+        end
         feature_gates = @config.kubelet&.feature_gates || {}
         if @config.cloud&.outtree_provider?
           feature_gates.merge!(@config.cloud.cloud_provider.feature_gates)

lib/pharos/kubeadm/kubeproxy_config.rb

@@ -17,6 +17,9 @@ def generate
           'kind' => 'KubeProxyConfiguration',
           'mode' => @config.kube_proxy&.mode || 'iptables'
         }
+        if @config.kube_proxy&.conntrack
+          config['conntrack'] = @config.kube_proxy.conntrack
+        end
 
         config
       end

lib/pharos/phases/configure_etcd.rb

@@ -32,7 +32,7 @@ def call
           'configure-etcd.sh',
           PEER_IP: @config.etcd_peer_address(@host),
           INITIAL_CLUSTER: initial_cluster.join(','),
-          IMAGE_REPO: @config.image_repository,
+          IMAGE_REPO: "k8s.gcr.io",
           ETCD_VERSION: Pharos::ETCD_VERSION,
           KUBE_VERSION: Pharos::KUBE_VERSION,
           ARCH: @host.cpu_arch.name,

lib/pharos/phases/configure_kubelet.rb

@@ -119,11 +119,6 @@ def build_systemd_dropin
 
         options << "ExecStartPre=-/sbin/swapoff -a"
 
-        if @host.resolvconf.systemd_resolved_stub
-          logger.info { "Adding POSTROUTING SNAT rule for systemd-resolved stub" }
-          options << "ExecStartPre=/bin/sh -c '/sbin/iptables -C POSTROUTING -t nat -d 127.0.0.53 -o lo -m comment --comment \"SNAT for systemd-resolved\" -j SNAT --to-source 127.0.0.1 || /sbin/iptables -I POSTROUTING -t nat -d 127.0.0.53 -o lo -m comment --comment \"SNAT for systemd-resolved\" -j SNAT --to-source 127.0.0.1'"
-        end
-
         "[Service]\n#{options.join("\n")}\n"
       end
 
@@ -132,11 +127,7 @@ def kubelet_extra_args
         args = []
         args += @host.kubelet_args(local_only: false, cloud_provider: @config.cloud&.provider)
 
-        if @host.resolvconf.systemd_resolved_stub
-          # use upstream resolvers instead of systemd stub resolver at localhost for `dnsPolicy: Default` pods
-          # XXX: kubeadm also handles this?
-          args << '--resolv-conf=/run/systemd/resolve/resolv.conf'
-        elsif @host.resolvconf.nameserver_localhost
+        if @host.resolvconf.nameserver_localhost
           fail "Host has /etc/resolv.conf configured with localhost as a resolver"
         end
 

lib/pharos/resources/node_local_dns/daemonset.yml.erb

@@ -26,7 +26,7 @@ spec:
         operator: Exists
       containers:
       - name: node-cache
-        image: <%= image_repository %>/k8s-dns-node-cache:<%= version %>
+        image: k8s.gcr.io/k8s-dns-node-cache:<%= version %>
         resources:
           limits:
             memory: 100Mi

lib/pharos/resources/weave/daemon-set.yml.erb

@@ -121,6 +121,7 @@ spec:
               mountPath: /run/xtables.lock
         <% end %>
       hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
       hostPID: true
       restartPolicy: Always
       securityContext:

lib/pharos/scripts/configure-etcd.sh

@@ -29,6 +29,7 @@ spec:
   - command:
     - etcd
     - --name=${PEER_NAME}
+    - --enable-v2=true
     - --cert-file=/etc/kubernetes/pki/etcd/server.pem
     - --key-file=/etc/kubernetes/pki/etcd/server-key.pem
     - --trusted-ca-file=/etc/kubernetes/pki/ca.pem