Install cri-o from pharos binary repo (#449)

kontena · Jun 26, 2018 · 67d5659 · 67d5659
1 parent dee67cd
commit 67d5659
Show file tree

Hide file tree

Showing 14 changed files with 233 additions and 59 deletions.
diff --git a/examples/vagrant/ubuntu/cluster.yml b/examples/vagrant/ubuntu/cluster.yml
@@ -1,6 +1,5 @@
 hosts:
   - address: 192.168.100.100
-    private_address: 192.168.100.100 # just to advertise correct ip with vagrant
     user: vagrant
     role: master
     ssh_key_path: ~/.vagrant.d/insecure_private_key
@@ -12,16 +11,9 @@ hosts:
     user: vagrant
     role: worker
     ssh_key_path: ~/.vagrant.d/insecure_private_key
-  - address: 192.168.100.103
-    user: vagrant
-    role: worker
-    ssh_key_path: ~/.vagrant.d/insecure_private_key
-  - address: 192.168.100.104
-    user: vagrant
-    role: worker
-    ssh_key_path: ~/.vagrant.d/insecure_private_key
 network:
   pod_network_cidr: 10.32.0.0/16
+  service_cidr: 10.33.0.0/16
   weave:
     trusted_subnets:
       - 192.168.100.0/24
@@ -30,11 +22,3 @@ addons:
     enabled: true
     configmap:
       map-hash-bucket-size: "128"
-  openebs:
-    enabled: true
-    default_storage_pool:
-      path: /var/openebs
-    default_storage_class:
-      replicas: 3
-      capacity: 5G
-      default_class: true
diff --git a/lib/pharos/configuration/host.rb b/lib/pharos/configuration/host.rb
@@ -44,7 +44,7 @@ def kubelet_args(local_only: false, cloud_provider: nil)
         if crio?
           args << '--container-runtime=remote'
           args << '--runtime-request-timeout=15m'
-          args << '--container-runtime-endpoint=/var/run/crio/crio.sock'
+          args << '--container-runtime-endpoint=unix:///var/run/crio/crio.sock'
         end
 
         if local_only

diff --git a/lib/pharos/host/el7/centos7.rb b/lib/pharos/host/el7/centos7.rb
@@ -15,6 +15,11 @@ class Centos7 < El7
         enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'docker' } }
       )
 
+      register_component(
+        name: 'cri-o', version: Pharos::CRIO_VERSION, license: 'Apache License 2.0',
+        enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'cri-o' } }
+      )
+
       register_component(
         name: 'cfssl', version: CFSSL_VERSION, license: 'MIT',
         enabled: proc { |c| !c.etcd&.endpoints }

diff --git a/lib/pharos/host/el7/el7.rb b/lib/pharos/host/el7/el7.rb
@@ -22,6 +22,7 @@ def install_essentials
 
       def configure_repos
         exec_script('repos/pharos_centos7.sh')
+        exec_script('repos/cri-o.sh') if crio?
       end
 
       def configure_netfilter
@@ -36,12 +37,22 @@ def configure_cfssl
       end
 
       def configure_container_runtime
-        raise Pharos::Error, "Unknown container runtime: #{host.container_runtime}" unless docker?
-
-        exec_script(
-          'configure-docker.sh',
-          DOCKER_VERSION: DOCKER_VERSION
-        )
+        if docker?
+          exec_script(
+            'configure-docker.sh',
+            DOCKER_VERSION: DOCKER_VERSION
+          )
+        elsif crio?
+          exec_script(
+            'configure-cri-o.sh',
+            CRIO_VERSION: Pharos::CRIO_VERSION,
+            CRIO_STREAM_ADDRESS: host.peer_address,
+            CPU_ARCH: host.cpu_arch.name,
+            IMAGE_REPO: cluster_config.image_repository
+          )
+        else
+          raise Pharos::Error, "Unknown container runtime: #{host.container_runtime}"
+        end
       end
 
       def ensure_kubelet(args)

diff --git a/lib/pharos/host/el7/rhel7.rb b/lib/pharos/host/el7/rhel7.rb
@@ -16,6 +16,11 @@ class Rhel7 < El7
         enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'docker' } }
       )
 
+      register_component(
+        name: 'cri-o', version: Pharos::CRIO_VERSION, license: 'Apache License 2.0',
+        enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'cri-o' } }
+      )
+
       register_component(
         name: 'cfssl', version: CFSSL_VERSION, license: 'MIT',
         enabled: proc { |c| !c.etcd&.endpoints }

diff --git a/lib/pharos/host/el7/scripts/configure-cri-o.sh b/lib/pharos/host/el7/scripts/configure-cri-o.sh
@@ -0,0 +1,93 @@
+#!/bin/sh
+
+set -e
+
+. /usr/local/share/pharos/util.sh
+
+yum install -y conntrack-tools libseccomp gpgme libassuan
+
+reload_daemon() {
+    if systemctl is-active --quiet crio; then
+        systemctl daemon-reload
+        systemctl restart crio
+    fi
+}
+
+tmpfile=$(mktemp /tmp/crio-service.XXXXXX)
+cat <<"EOF" >${tmpfile}
+[Unit]
+Description=Open Container Initiative Daemon
+Documentation=https://github.com/kubernetes-incubator/cri-o
+After=network-online.target
+
+[Service]
+Type=notify
+Environment=GOTRACEBACK=crash
+ExecStartPre=/sbin/sysctl -w net.ipv4.ip_forward=1
+ExecStart=/usr/local/bin/crio \
+          $CRIO_STORAGE_OPTIONS \
+          $CRIO_NETWORK_OPTIONS
+ExecReload=/bin/kill -s HUP $MAINPID
+TasksMax=infinity
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+OOMScoreAdjust=-999
+TimeoutStartSec=0
+Restart=on-abnormal
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+if diff $tmpfile /etc/systemd/system/crio.service > /dev/null ; then
+    rm $tmpfile
+else
+    mv $tmpfile /etc/systemd/system/crio.service
+fi
+
+mkdir -p /etc/systemd/system/crio.service.d
+if [ -n "$HTTP_PROXY" ]; then
+    cat <<EOF >/etc/systemd/system/crio.service.d/http-proxy.conf
+[Service]
+Environment="HTTP_PROXY=${HTTP_PROXY}"
+EOF
+    reload_daemon
+else
+    if [ -f /etc/systemd/system/crio.service.d/http-proxy.conf ]; then
+        rm /etc/systemd/system/crio.service.d/http-proxy.conf
+        reload_daemon
+    fi
+fi
+
+if [ ! "$(cat /etc/crio/.version)" = "$CRIO_VERSION" ]; then
+    DL_URL="https://dl.bintray.com/kontena/pharos-bin/cri-o/cri-o-v${CRIO_VERSION}-linux-amd64.tar.gz"
+    curl -sSL $DL_URL -o /tmp/cri-o.tar.gz
+    curl -sSL "${DL_URL}.asc" -o /tmp/cri-o.tar.gz.asc
+    gpg --verify /tmp/cri-o.tar.gz.asc /tmp/cri-o.tar.gz
+    tar -C / -xzf /tmp/cri-o.tar.gz
+    rm /tmp/cri-o.tar.gz /tmp/cri-o.tar.gz.asc
+    echo $CRIO_VERSION > /etc/crio/.version 
+fi
+
+rm -f /etc/cni/net.d/100-crio-bridge.conf /etc/cni/net.d/200-loopback.conf || true
+
+orig_config=$(cat /etc/crio/crio.conf)
+lineinfile "^stream_address =" "stream_address = \"${CRIO_STREAM_ADDRESS}\"" "/etc/crio/crio.conf"
+lineinfile "^cgroup_manager =" "cgroup_manager = \"systemd\"" "/etc/crio/crio.conf"
+lineinfile "^log_size_max =" "log_size_max = 134217728" "/etc/crio/crio.conf"
+lineinfile "^pause_image =" "pause_image = \"${IMAGE_REPO}/pause-${CPU_ARCH}:3.1\"" "/etc/crio/crio.conf"
+
+if ! systemctl is-active --quiet crio; then
+    systemctl daemon-reload
+    systemctl enable crio
+    systemctl start crio
+else 
+    if systemctl status crio 2>&1 | grep -q 'changed on disk' ; then
+        reload_daemon
+    fi
+
+    if [ "$orig_config" != "$(cat /etc/crio/crio.conf)" ]; then
+        reload_daemon
+    fi
+fi
diff --git a/lib/pharos/host/el7/scripts/repos/cri-o.sh b/lib/pharos/host/el7/scripts/repos/cri-o.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -eu
+
+# Add bintray key to gpg
+curl -fsSL https://bintray.com/user/downloadSubjectPublicKey?username=bintray | gpg --import
diff --git a/lib/pharos/host/ubuntu/scripts/configure-cri-o.sh b/lib/pharos/host/ubuntu/scripts/configure-cri-o.sh
@@ -1,6 +1,8 @@
 #!/bin/sh
 
-set -ue
+set -e
+
+. /usr/local/share/pharos/util.sh
 
 reload_daemon() {
     if systemctl is-active --quiet crio; then
@@ -9,13 +11,51 @@ reload_daemon() {
     fi
 }
 
-mkdir -p /etc/systemd/system/crio.service.d
-cat <<EOF >/etc/systemd/system/crio.service.d/10-cgroup.conf
+export DEBIAN_FRONTEND=noninteractive 
+if dpkg -s cri-o-1.10 ; then
+    systemctl stop crio
+    systemctl disable crio
+    apt-get remove -y --purge cri-o-1.10
+    rm /etc/systemd/system/crio.service.d/10-cgroup.conf || true
+    systemctl daemon-reload
+fi
+apt-get install -y conntrack libgpgme11 libseccomp2 libassuan0
+
+tmpfile=$(mktemp /tmp/crio-service.XXXXXX)
+cat <<"EOF" >${tmpfile}
+[Unit]
+Description=Open Container Initiative Daemon
+Documentation=https://github.com/kubernetes-incubator/cri-o
+After=network-online.target
+
 [Service]
-Environment='CRIO_STORAGE_OPTIONS=--cgroup-manager=cgroupfs --stream-address=$CRIO_STREAM_ADDRESS --pause-image=${IMAGE_REPO}/pause-${CPU_ARCH}:3.1'
+Type=notify
+Environment=GOTRACEBACK=crash
 ExecStartPre=/sbin/sysctl -w net.ipv4.ip_forward=1
+ExecStart=/usr/local/bin/crio \
+          $CRIO_STORAGE_OPTIONS \
+          $CRIO_NETWORK_OPTIONS
+ExecReload=/bin/kill -s HUP $MAINPID
+TasksMax=infinity
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+OOMScoreAdjust=-999
+TimeoutStartSec=0
+Restart=on-abnormal
+
+[Install]
+WantedBy=multi-user.target
 EOF
 
+if diff $tmpfile /etc/systemd/system/crio.service > /dev/null ; then
+    rm $tmpfile
+else
+    mv $tmpfile /etc/systemd/system/crio.service
+fi
+
+mkdir -p /etc/systemd/system/crio.service.d
+
 if [ -n "$HTTP_PROXY" ]; then
     cat <<EOF >/etc/systemd/system/crio.service.d/http-proxy.conf
 [Service]
@@ -29,20 +69,34 @@ else
     fi
 fi
 
-DEBIAN_FRONTEND=noninteractive apt-get install -y cri-o-$CRIO_VERSION
-systemctl enable crio
-# remove unnecessary cni plugins
-rm /etc/cni/net.d/100-crio-bridge.conf /etc/cni/net.d/200-loopback.conf || true
-systemctl start crio
+if [ ! "$(cat /etc/crio/.version)" = "$CRIO_VERSION" ]; then
+    DL_URL="https://dl.bintray.com/kontena/pharos-bin/cri-o/cri-o-v${CRIO_VERSION}-linux-amd64.tar.gz"
+    curl -sSL $DL_URL -o /tmp/cri-o.tar.gz
+    curl -sSL "${DL_URL}.asc" -o /tmp/cri-o.tar.gz.asc
+    gpg --verify /tmp/cri-o.tar.gz.asc /tmp/cri-o.tar.gz
+    tar -C / -xzf /tmp/cri-o.tar.gz
+    rm /tmp/cri-o.tar.gz /tmp/cri-o.tar.gz.asc
+    echo $CRIO_VERSION > /etc/crio/.version 
+fi
 
-# Install crictl binary if needed
+rm -f /etc/cni/net.d/100-crio-bridge.conf /etc/cni/net.d/200-loopback.conf || true
 
+orig_config=$(cat /etc/crio/crio.conf)
+lineinfile "^stream_address =" "stream_address = \"${CRIO_STREAM_ADDRESS}\"" "/etc/crio/crio.conf"
+lineinfile "^cgroup_manager =" "cgroup_manager = \"cgroupfs\"" "/etc/crio/crio.conf"
+lineinfile "^log_size_max =" "log_size_max = 134217728" "/etc/crio/crio.conf"
+lineinfile "^pause_image =" "pause_image = \"${IMAGE_REPO}\/pause-${CPU_ARCH}:3.1\"" "/etc/crio/crio.conf"
 
-if ! which crictl > /dev/null || [ "$(crictl -v)" != "$CRICTL_VERSION" ]; then
-    # Not installed or wrong version
-    curl -sSL https://bintray.com/kontena/pharos-bin/download_file?file_path=crictl-v${CRICTL_VERSION}-linux-${CPU_ARCH}.tar.gz -o /tmp/crictl.tar.gz
-    curl -sSL https://bintray.com/kontena/pharos-bin/download_file?file_path=crictl-v${CRICTL_VERSION}-linux-${CPU_ARCH}.tar.gz.asc -o /tmp/crictl.tar.gz.asc
-    gpg --verify /tmp/crictl.tar.gz.asc /tmp/crictl.tar.gz
-    tar xzf /tmp/crictl.tar.gz
-    install -m 755 -o root -g root crictl /usr/local/bin/crictl && rm crictl
-fi
+if ! systemctl is-active --quiet crio; then
+    systemctl daemon-reload
+    systemctl enable crio
+    systemctl start crio
+else 
+    if systemctl status crio 2>&1 | grep -q 'changed on disk' ; then
+        reload_daemon
+    fi
+
+    if [ "$orig_config" != "$(cat /etc/crio/crio.conf)" ]; then
+        reload_daemon
+    fi
+fi
diff --git a/lib/pharos/host/ubuntu/scripts/repos/cri-o.sh b/lib/pharos/host/ubuntu/scripts/repos/cri-o.sh
@@ -2,9 +2,11 @@
 
 set -eu
 
-if [ ! -e /etc/apt/sources.list.d/projectatomic-ubuntu-ppa-xenial.list ]; then
-    add-apt-repository ppa:projectatomic/ppa
-fi
-
 # Add bintray key to gpg
 curl -fsSL https://bintray.com/user/downloadSubjectPublicKey?username=bintray | gpg --import
+
+# remove deprecated ppa repository
+if [ -e /etc/apt/sources.list.d/projectatomic-ubuntu-ppa-xenial.list ]; then	
+    add-apt-repository --remove ppa:projectatomic/ppa
+    rm /etc/apt/sources.list.d/projectatomic-ubuntu-ppa-xenial.list
+fi
diff --git a/lib/pharos/host/ubuntu/ubuntu_bionic.rb b/lib/pharos/host/ubuntu/ubuntu_bionic.rb
@@ -15,24 +15,40 @@ class UbuntuBionic < Ubuntu
         enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'docker' } }
       )
 
+      register_component(
+        name: 'cri-o', version: DOCKER_VERSION, license: 'Apache License 2.0',
+        enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'cri-o' } }
+      )
+
       register_component(
         name: 'cfssl', version: CFSSL_VERSION, license: 'MIT',
         enabled: proc { |c| !c.etcd&.endpoints }
       )
 
       def configure_repos
+        exec_script('repos/cri-o.sh') if crio?
         exec_script("repos/pharos_bionic.sh")
         exec_script('repos/update.sh')
       end
 
       def configure_container_runtime
-        raise Pharos::Error, "Unknown container runtime: #{host.container_runtime}" unless docker?
-
-        exec_script(
-          'configure-docker.sh',
-          DOCKER_PACKAGE: 'docker.io',
-          DOCKER_VERSION: "#{DOCKER_VERSION}-0ubuntu1"
-        )
+        if docker?
+          exec_script(
+            'configure-docker.sh',
+            DOCKER_PACKAGE: 'docker.io',
+            DOCKER_VERSION: "#{DOCKER_VERSION}-0ubuntu1~16.04.2"
+          )
+        elsif crio?
+          exec_script(
+            'configure-cri-o.sh',
+            CRIO_VERSION: Pharos::CRIO_VERSION,
+            CRIO_STREAM_ADDRESS: host.peer_address,
+            CPU_ARCH: host.cpu_arch.name,
+            IMAGE_REPO: cluster_config.image_repository
+          )
+        else
+          raise Pharos::Error, "Unknown container runtime: #{host.container_runtime}"
+        end
       end
     end
   end