diff --git a/examples/vagrant/ubuntu/cluster.yml b/examples/vagrant/ubuntu/cluster.yml index b4d994bd6..7acdbeac2 100644 --- a/examples/vagrant/ubuntu/cluster.yml +++ b/examples/vagrant/ubuntu/cluster.yml @@ -1,6 +1,5 @@ hosts: - address: 192.168.100.100 - private_address: 192.168.100.100 # just to advertise correct ip with vagrant user: vagrant role: master ssh_key_path: ~/.vagrant.d/insecure_private_key @@ -12,16 +11,9 @@ hosts: user: vagrant role: worker ssh_key_path: ~/.vagrant.d/insecure_private_key - - address: 192.168.100.103 - user: vagrant - role: worker - ssh_key_path: ~/.vagrant.d/insecure_private_key - - address: 192.168.100.104 - user: vagrant - role: worker - ssh_key_path: ~/.vagrant.d/insecure_private_key network: pod_network_cidr: 10.32.0.0/16 + service_cidr: 10.33.0.0/16 weave: trusted_subnets: - 192.168.100.0/24 @@ -30,11 +22,3 @@ addons: enabled: true configmap: map-hash-bucket-size: "128" - openebs: - enabled: true - default_storage_pool: - path: /var/openebs - default_storage_class: - replicas: 3 - capacity: 5G - default_class: true diff --git a/lib/pharos/configuration/host.rb b/lib/pharos/configuration/host.rb index 369da8d91..28053d72e 100644 --- a/lib/pharos/configuration/host.rb +++ b/lib/pharos/configuration/host.rb @@ -44,7 +44,7 @@ def kubelet_args(local_only: false, cloud_provider: nil) if crio? args << '--container-runtime=remote' args << '--runtime-request-timeout=15m' - args << '--container-runtime-endpoint=/var/run/crio/crio.sock' + args << '--container-runtime-endpoint=unix:///var/run/crio/crio.sock' end if local_only diff --git a/lib/pharos/host/el7/centos7.rb b/lib/pharos/host/el7/centos7.rb index 3cf5ada06..46928cf4c 100644 --- a/lib/pharos/host/el7/centos7.rb +++ b/lib/pharos/host/el7/centos7.rb @@ -15,6 +15,11 @@ class Centos7 < El7 enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'docker' } } ) + register_component( + name: 'cri-o', version: Pharos::CRIO_VERSION, license: 'Apache License 2.0', + enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'cri-o' } } + ) + register_component( name: 'cfssl', version: CFSSL_VERSION, license: 'MIT', enabled: proc { |c| !c.etcd&.endpoints } diff --git a/lib/pharos/host/el7/el7.rb b/lib/pharos/host/el7/el7.rb index f035cebc8..3c37f29e8 100644 --- a/lib/pharos/host/el7/el7.rb +++ b/lib/pharos/host/el7/el7.rb @@ -22,6 +22,7 @@ def install_essentials def configure_repos exec_script('repos/pharos_centos7.sh') + exec_script('repos/cri-o.sh') if crio? end def configure_netfilter @@ -36,12 +37,22 @@ def configure_cfssl end def configure_container_runtime - raise Pharos::Error, "Unknown container runtime: #{host.container_runtime}" unless docker? - - exec_script( - 'configure-docker.sh', - DOCKER_VERSION: DOCKER_VERSION - ) + if docker? + exec_script( + 'configure-docker.sh', + DOCKER_VERSION: DOCKER_VERSION + ) + elsif crio? + exec_script( + 'configure-cri-o.sh', + CRIO_VERSION: Pharos::CRIO_VERSION, + CRIO_STREAM_ADDRESS: host.peer_address, + CPU_ARCH: host.cpu_arch.name, + IMAGE_REPO: cluster_config.image_repository + ) + else + raise Pharos::Error, "Unknown container runtime: #{host.container_runtime}" + end end def ensure_kubelet(args) diff --git a/lib/pharos/host/el7/rhel7.rb b/lib/pharos/host/el7/rhel7.rb index b3554ca2d..a57b79643 100644 --- a/lib/pharos/host/el7/rhel7.rb +++ b/lib/pharos/host/el7/rhel7.rb @@ -16,6 +16,11 @@ class Rhel7 < El7 enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'docker' } } ) + register_component( + name: 'cri-o', version: Pharos::CRIO_VERSION, license: 'Apache License 2.0', + enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'cri-o' } } + ) + register_component( name: 'cfssl', version: CFSSL_VERSION, license: 'MIT', enabled: proc { |c| !c.etcd&.endpoints } diff --git a/lib/pharos/host/el7/scripts/configure-cri-o.sh b/lib/pharos/host/el7/scripts/configure-cri-o.sh new file mode 100755 index 000000000..dd6d23a32 --- /dev/null +++ b/lib/pharos/host/el7/scripts/configure-cri-o.sh @@ -0,0 +1,93 @@ +#!/bin/sh + +set -e + +. /usr/local/share/pharos/util.sh + +yum install -y conntrack-tools libseccomp gpgme libassuan + +reload_daemon() { + if systemctl is-active --quiet crio; then + systemctl daemon-reload + systemctl restart crio + fi +} + +tmpfile=$(mktemp /tmp/crio-service.XXXXXX) +cat <<"EOF" >${tmpfile} +[Unit] +Description=Open Container Initiative Daemon +Documentation=https://github.com/kubernetes-incubator/cri-o +After=network-online.target + +[Service] +Type=notify +Environment=GOTRACEBACK=crash +ExecStartPre=/sbin/sysctl -w net.ipv4.ip_forward=1 +ExecStart=/usr/local/bin/crio \ + $CRIO_STORAGE_OPTIONS \ + $CRIO_NETWORK_OPTIONS +ExecReload=/bin/kill -s HUP $MAINPID +TasksMax=infinity +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +OOMScoreAdjust=-999 +TimeoutStartSec=0 +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target +EOF + +if diff $tmpfile /etc/systemd/system/crio.service > /dev/null ; then + rm $tmpfile +else + mv $tmpfile /etc/systemd/system/crio.service +fi + +mkdir -p /etc/systemd/system/crio.service.d +if [ -n "$HTTP_PROXY" ]; then + cat </etc/systemd/system/crio.service.d/http-proxy.conf +[Service] +Environment="HTTP_PROXY=${HTTP_PROXY}" +EOF + reload_daemon +else + if [ -f /etc/systemd/system/crio.service.d/http-proxy.conf ]; then + rm /etc/systemd/system/crio.service.d/http-proxy.conf + reload_daemon + fi +fi + +if [ ! "$(cat /etc/crio/.version)" = "$CRIO_VERSION" ]; then + DL_URL="https://dl.bintray.com/kontena/pharos-bin/cri-o/cri-o-v${CRIO_VERSION}-linux-amd64.tar.gz" + curl -sSL $DL_URL -o /tmp/cri-o.tar.gz + curl -sSL "${DL_URL}.asc" -o /tmp/cri-o.tar.gz.asc + gpg --verify /tmp/cri-o.tar.gz.asc /tmp/cri-o.tar.gz + tar -C / -xzf /tmp/cri-o.tar.gz + rm /tmp/cri-o.tar.gz /tmp/cri-o.tar.gz.asc + echo $CRIO_VERSION > /etc/crio/.version +fi + +rm -f /etc/cni/net.d/100-crio-bridge.conf /etc/cni/net.d/200-loopback.conf || true + +orig_config=$(cat /etc/crio/crio.conf) +lineinfile "^stream_address =" "stream_address = \"${CRIO_STREAM_ADDRESS}\"" "/etc/crio/crio.conf" +lineinfile "^cgroup_manager =" "cgroup_manager = \"systemd\"" "/etc/crio/crio.conf" +lineinfile "^log_size_max =" "log_size_max = 134217728" "/etc/crio/crio.conf" +lineinfile "^pause_image =" "pause_image = \"${IMAGE_REPO}/pause-${CPU_ARCH}:3.1\"" "/etc/crio/crio.conf" + +if ! systemctl is-active --quiet crio; then + systemctl daemon-reload + systemctl enable crio + systemctl start crio +else + if systemctl status crio 2>&1 | grep -q 'changed on disk' ; then + reload_daemon + fi + + if [ "$orig_config" != "$(cat /etc/crio/crio.conf)" ]; then + reload_daemon + fi +fi \ No newline at end of file diff --git a/lib/pharos/host/el7/scripts/repos/cri-o.sh b/lib/pharos/host/el7/scripts/repos/cri-o.sh new file mode 100644 index 000000000..d736e90a7 --- /dev/null +++ b/lib/pharos/host/el7/scripts/repos/cri-o.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -eu + +# Add bintray key to gpg +curl -fsSL https://bintray.com/user/downloadSubjectPublicKey?username=bintray | gpg --import diff --git a/lib/pharos/host/ubuntu/scripts/configure-cri-o.sh b/lib/pharos/host/ubuntu/scripts/configure-cri-o.sh index 582f926e0..b1b829e07 100755 --- a/lib/pharos/host/ubuntu/scripts/configure-cri-o.sh +++ b/lib/pharos/host/ubuntu/scripts/configure-cri-o.sh @@ -1,6 +1,8 @@ #!/bin/sh -set -ue +set -e + +. /usr/local/share/pharos/util.sh reload_daemon() { if systemctl is-active --quiet crio; then @@ -9,13 +11,51 @@ reload_daemon() { fi } -mkdir -p /etc/systemd/system/crio.service.d -cat </etc/systemd/system/crio.service.d/10-cgroup.conf +export DEBIAN_FRONTEND=noninteractive +if dpkg -s cri-o-1.10 ; then + systemctl stop crio + systemctl disable crio + apt-get remove -y --purge cri-o-1.10 + rm /etc/systemd/system/crio.service.d/10-cgroup.conf || true + systemctl daemon-reload +fi +apt-get install -y conntrack libgpgme11 libseccomp2 libassuan0 + +tmpfile=$(mktemp /tmp/crio-service.XXXXXX) +cat <<"EOF" >${tmpfile} +[Unit] +Description=Open Container Initiative Daemon +Documentation=https://github.com/kubernetes-incubator/cri-o +After=network-online.target + [Service] -Environment='CRIO_STORAGE_OPTIONS=--cgroup-manager=cgroupfs --stream-address=$CRIO_STREAM_ADDRESS --pause-image=${IMAGE_REPO}/pause-${CPU_ARCH}:3.1' +Type=notify +Environment=GOTRACEBACK=crash ExecStartPre=/sbin/sysctl -w net.ipv4.ip_forward=1 +ExecStart=/usr/local/bin/crio \ + $CRIO_STORAGE_OPTIONS \ + $CRIO_NETWORK_OPTIONS +ExecReload=/bin/kill -s HUP $MAINPID +TasksMax=infinity +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +OOMScoreAdjust=-999 +TimeoutStartSec=0 +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target EOF +if diff $tmpfile /etc/systemd/system/crio.service > /dev/null ; then + rm $tmpfile +else + mv $tmpfile /etc/systemd/system/crio.service +fi + +mkdir -p /etc/systemd/system/crio.service.d + if [ -n "$HTTP_PROXY" ]; then cat </etc/systemd/system/crio.service.d/http-proxy.conf [Service] @@ -29,20 +69,34 @@ else fi fi -DEBIAN_FRONTEND=noninteractive apt-get install -y cri-o-$CRIO_VERSION -systemctl enable crio -# remove unnecessary cni plugins -rm /etc/cni/net.d/100-crio-bridge.conf /etc/cni/net.d/200-loopback.conf || true -systemctl start crio +if [ ! "$(cat /etc/crio/.version)" = "$CRIO_VERSION" ]; then + DL_URL="https://dl.bintray.com/kontena/pharos-bin/cri-o/cri-o-v${CRIO_VERSION}-linux-amd64.tar.gz" + curl -sSL $DL_URL -o /tmp/cri-o.tar.gz + curl -sSL "${DL_URL}.asc" -o /tmp/cri-o.tar.gz.asc + gpg --verify /tmp/cri-o.tar.gz.asc /tmp/cri-o.tar.gz + tar -C / -xzf /tmp/cri-o.tar.gz + rm /tmp/cri-o.tar.gz /tmp/cri-o.tar.gz.asc + echo $CRIO_VERSION > /etc/crio/.version +fi -# Install crictl binary if needed +rm -f /etc/cni/net.d/100-crio-bridge.conf /etc/cni/net.d/200-loopback.conf || true +orig_config=$(cat /etc/crio/crio.conf) +lineinfile "^stream_address =" "stream_address = \"${CRIO_STREAM_ADDRESS}\"" "/etc/crio/crio.conf" +lineinfile "^cgroup_manager =" "cgroup_manager = \"cgroupfs\"" "/etc/crio/crio.conf" +lineinfile "^log_size_max =" "log_size_max = 134217728" "/etc/crio/crio.conf" +lineinfile "^pause_image =" "pause_image = \"${IMAGE_REPO}\/pause-${CPU_ARCH}:3.1\"" "/etc/crio/crio.conf" -if ! which crictl > /dev/null || [ "$(crictl -v)" != "$CRICTL_VERSION" ]; then - # Not installed or wrong version - curl -sSL https://bintray.com/kontena/pharos-bin/download_file?file_path=crictl-v${CRICTL_VERSION}-linux-${CPU_ARCH}.tar.gz -o /tmp/crictl.tar.gz - curl -sSL https://bintray.com/kontena/pharos-bin/download_file?file_path=crictl-v${CRICTL_VERSION}-linux-${CPU_ARCH}.tar.gz.asc -o /tmp/crictl.tar.gz.asc - gpg --verify /tmp/crictl.tar.gz.asc /tmp/crictl.tar.gz - tar xzf /tmp/crictl.tar.gz - install -m 755 -o root -g root crictl /usr/local/bin/crictl && rm crictl -fi +if ! systemctl is-active --quiet crio; then + systemctl daemon-reload + systemctl enable crio + systemctl start crio +else + if systemctl status crio 2>&1 | grep -q 'changed on disk' ; then + reload_daemon + fi + + if [ "$orig_config" != "$(cat /etc/crio/crio.conf)" ]; then + reload_daemon + fi +fi \ No newline at end of file diff --git a/lib/pharos/host/ubuntu/scripts/repos/cri-o.sh b/lib/pharos/host/ubuntu/scripts/repos/cri-o.sh index a76a89e54..88a26b955 100755 --- a/lib/pharos/host/ubuntu/scripts/repos/cri-o.sh +++ b/lib/pharos/host/ubuntu/scripts/repos/cri-o.sh @@ -2,9 +2,11 @@ set -eu -if [ ! -e /etc/apt/sources.list.d/projectatomic-ubuntu-ppa-xenial.list ]; then - add-apt-repository ppa:projectatomic/ppa -fi - # Add bintray key to gpg curl -fsSL https://bintray.com/user/downloadSubjectPublicKey?username=bintray | gpg --import + +# remove deprecated ppa repository +if [ -e /etc/apt/sources.list.d/projectatomic-ubuntu-ppa-xenial.list ]; then + add-apt-repository --remove ppa:projectatomic/ppa + rm /etc/apt/sources.list.d/projectatomic-ubuntu-ppa-xenial.list +fi \ No newline at end of file diff --git a/lib/pharos/host/ubuntu/ubuntu_bionic.rb b/lib/pharos/host/ubuntu/ubuntu_bionic.rb index 3714daaa4..6d4b846df 100644 --- a/lib/pharos/host/ubuntu/ubuntu_bionic.rb +++ b/lib/pharos/host/ubuntu/ubuntu_bionic.rb @@ -15,24 +15,40 @@ class UbuntuBionic < Ubuntu enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'docker' } } ) + register_component( + name: 'cri-o', version: DOCKER_VERSION, license: 'Apache License 2.0', + enabled: proc { |c| c.hosts.any? { |h| h.container_runtime == 'cri-o' } } + ) + register_component( name: 'cfssl', version: CFSSL_VERSION, license: 'MIT', enabled: proc { |c| !c.etcd&.endpoints } ) def configure_repos + exec_script('repos/cri-o.sh') if crio? exec_script("repos/pharos_bionic.sh") exec_script('repos/update.sh') end def configure_container_runtime - raise Pharos::Error, "Unknown container runtime: #{host.container_runtime}" unless docker? - - exec_script( - 'configure-docker.sh', - DOCKER_PACKAGE: 'docker.io', - DOCKER_VERSION: "#{DOCKER_VERSION}-0ubuntu1" - ) + if docker? + exec_script( + 'configure-docker.sh', + DOCKER_PACKAGE: 'docker.io', + DOCKER_VERSION: "#{DOCKER_VERSION}-0ubuntu1~16.04.2" + ) + elsif crio? + exec_script( + 'configure-cri-o.sh', + CRIO_VERSION: Pharos::CRIO_VERSION, + CRIO_STREAM_ADDRESS: host.peer_address, + CPU_ARCH: host.cpu_arch.name, + IMAGE_REPO: cluster_config.image_repository + ) + else + raise Pharos::Error, "Unknown container runtime: #{host.container_runtime}" + end end end end diff --git a/lib/pharos/host/ubuntu/ubuntu_xenial.rb b/lib/pharos/host/ubuntu/ubuntu_xenial.rb index 2a593bf07..3cb057dbc 100644 --- a/lib/pharos/host/ubuntu/ubuntu_xenial.rb +++ b/lib/pharos/host/ubuntu/ubuntu_xenial.rb @@ -7,7 +7,6 @@ module Host class UbuntuXenial < Ubuntu register_config 'ubuntu', '16.04' - CRIO_VERSION = '1.10' DOCKER_VERSION = '1.13.1' CFSSL_VERSION = '1.2' @@ -42,8 +41,7 @@ def configure_container_runtime elsif crio? exec_script( 'configure-cri-o.sh', - CRIO_VERSION: CRIO_VERSION, - CRICTL_VERSION: Pharos::CRICTL_VERSION, + CRIO_VERSION: Pharos::CRIO_VERSION, CRIO_STREAM_ADDRESS: host.peer_address, CPU_ARCH: host.cpu_arch.name, IMAGE_REPO: cluster_config.image_repository diff --git a/lib/pharos/phases/configure_host.rb b/lib/pharos/phases/configure_host.rb index d88e41d40..8c864ddb3 100644 --- a/lib/pharos/phases/configure_host.rb +++ b/lib/pharos/phases/configure_host.rb @@ -18,7 +18,7 @@ def call logger.info { "Configuring netfilter ..." } host_configurer.configure_netfilter - logger.info { "Configuring container runtime (docker) packages ..." } + logger.info { "Configuring container runtime (#{@host.container_runtime}) packages ..." } host_configurer.configure_container_runtime end end diff --git a/lib/pharos/scripts/pharos.sh b/lib/pharos/scripts/pharos.sh index 507b5c5b7..d242be235 100755 --- a/lib/pharos/scripts/pharos.sh +++ b/lib/pharos/scripts/pharos.sh @@ -21,7 +21,7 @@ lineinfile() { for file in "$@"; do file_exists "$file" || return 1 - grep -q "${match}" $file && sed "s/${match}.*/${line}/" -i $file || echo $line >> $file + grep -q "${match}" $file && sed "s|${match}.*|${line}|" -i $file || echo $line >> $file done return 0 diff --git a/lib/pharos_cluster.rb b/lib/pharos_cluster.rb index 19b3aa3b4..c6f3ed5e2 100644 --- a/lib/pharos_cluster.rb +++ b/lib/pharos_cluster.rb @@ -8,7 +8,7 @@ require_relative "pharos/root_command" module Pharos - CRIO_VERSION = '1.10' + CRIO_VERSION = '1.10.3' CRICTL_VERSION = '1.0.0-beta.0' KUBE_VERSION = ENV.fetch('KUBE_VERSION') { '1.10.4' } KUBEADM_VERSION = ENV.fetch('KUBEADM_VERSION') { KUBE_VERSION }