diff --git a/instances/k8smaster/cloud_init/bootstrap.template.yaml b/instances/k8smaster/cloud_init/bootstrap.template.yaml index 1bb8606..739b2ec 100644 --- a/instances/k8smaster/cloud_init/bootstrap.template.yaml +++ b/instances/k8smaster/cloud_init/bootstrap.template.yaml @@ -79,6 +79,12 @@ write_files: content: | ${kubelet_service_content} + - path: "/root/services/ethtool-disable-offload@.service" + permissions: "0600" + encoding: "gzip+base64" + content: | + ${ethtool_service_content} + # Kube certs, tokens - path: "/etc/kubernetes/ssl/ca.pem" permissions: "0600" diff --git a/instances/k8smaster/datasources.tf b/instances/k8smaster/datasources.tf index 6a81a1b..606b616 100644 --- a/instances/k8smaster/datasources.tf +++ b/instances/k8smaster/datasources.tf @@ -57,6 +57,10 @@ data "template_file" "kubelet-service" { } } +data "template_file" "ethtool-offload-service" { + template = "${file("${path.module}/scripts/ethtool-disable-offload.service")}" +} + data "template_file" "kube-controller-manager" { template = "${file("${path.module}/manifests/kube-controller-manager.yaml")}" @@ -132,6 +136,7 @@ data "template_file" "kube_master_cloud_init_file" { master_kubeconfig_template_content = "${base64gzip(data.template_file.master-kubeconfig.rendered)}" kube_scheduler_template_content = "${base64gzip(data.template_file.kube-scheduler.rendered)}" kubelet_service_content = "${base64gzip(data.template_file.kubelet-service.rendered)}" + ethtool_service_content = "${base64gzip(data.template_file.ethtool-offload-service.rendered)}" ca-pem-content = "${base64gzip(var.root_ca_pem)}" ca-key-content = "${base64gzip(var.root_ca_key)}" api-server-key-content = "${base64gzip(var.api_server_private_key_pem)}" diff --git a/instances/k8smaster/scripts/ethtool-disable-offload.service b/instances/k8smaster/scripts/ethtool-disable-offload.service new file mode 100644 index 0000000..7713e4c --- /dev/null +++ b/instances/k8smaster/scripts/ethtool-disable-offload.service @@ -0,0 +1,12 @@ +[Unit] +Description=ethtool-disable-offload +Requires=network.target +After=network.target +Before=kubelet.service + +[Service] +ExecStart=/sbin/ethtool -K %i tx off +Type=oneshot + +[Install] +WantedBy=multi-user.target diff --git a/instances/k8smaster/scripts/setup.template.sh b/instances/k8smaster/scripts/setup.template.sh index e9b21e0..3cd4742 100644 --- a/instances/k8smaster/scripts/setup.template.sh +++ b/instances/k8smaster/scripts/setup.template.sh @@ -32,7 +32,11 @@ fi BROADCOM_DRIVER=$(lsmod | grep bnxt_en | awk '{print $1}') if [[ -n "$${BROADCOM_DRIVER}" ]]; then echo "Disabling hardware TX checksum offloading" - ethtool --offload $(ip -o -4 route show to default | awk '{print $5}') tx off + mv /root/services/ethtool-disable-offload@.service /etc/systemd/system + systemctl daemon-reload + DEV=$(ip -o -4 route show to default | awk '{print $5}') + systemctl enable ethtool-disable-offload@$DEV + systemctl start ethtool-disable-offload@$DEV fi # Download etcdctl client diff --git a/instances/k8sworker/cloud_init/bootstrap.template.yaml b/instances/k8sworker/cloud_init/bootstrap.template.yaml index b445eb1..ec5c2d2 100644 --- a/instances/k8sworker/cloud_init/bootstrap.template.yaml +++ b/instances/k8sworker/cloud_init/bootstrap.template.yaml @@ -36,6 +36,12 @@ write_files: content: | ${kubelet_service_content} + - path: "/root/services/ethtool-disable-offload@.service" + permissions: "0600" + encoding: "gzip+base64" + content: | + ${ethtool_service_content} + # Kube certs - path: "/etc/kubernetes/ssl/ca.pem" permissions: "0600" diff --git a/instances/k8sworker/datasources.tf b/instances/k8sworker/datasources.tf index 72058a5..4ddd6f5 100644 --- a/instances/k8sworker/datasources.tf +++ b/instances/k8sworker/datasources.tf @@ -66,6 +66,10 @@ data "template_file" "kubelet-service" { } } +data "template_file" "ethtool-offload-service" { + template = "${file("${path.module}/scripts/ethtool-disable-offload.service")}" +} + data "template_file" "kube_worker_cloud_init_file" { template = "${file("${path.module}/cloud_init/bootstrap.template.yaml")}" @@ -76,6 +80,7 @@ data "template_file" "kube_worker_cloud_init_file" { kube_proxy_template_content = "${base64gzip(data.template_file.kube-proxy.rendered)}" worker_kubeconfig_template_content = "${base64gzip(data.template_file.worker-kubeconfig.rendered)}" kubelet_service_content = "${base64gzip(data.template_file.kubelet-service.rendered)}" + ethtool_service_content = "${base64gzip(data.template_file.ethtool-offload-service.rendered)}" ca-pem-content = "${base64gzip(var.root_ca_pem)}" ca-key-content = "${base64gzip(var.root_ca_key)}" api-server-key-content = "${base64gzip(var.api_server_private_key_pem)}" diff --git a/instances/k8sworker/scripts/ethtool-disable-offload.service b/instances/k8sworker/scripts/ethtool-disable-offload.service new file mode 100644 index 0000000..7713e4c --- /dev/null +++ b/instances/k8sworker/scripts/ethtool-disable-offload.service @@ -0,0 +1,12 @@ +[Unit] +Description=ethtool-disable-offload +Requires=network.target +After=network.target +Before=kubelet.service + +[Service] +ExecStart=/sbin/ethtool -K %i tx off +Type=oneshot + +[Install] +WantedBy=multi-user.target diff --git a/instances/k8sworker/scripts/setup.template.sh b/instances/k8sworker/scripts/setup.template.sh index 6122d27..14d846a 100644 --- a/instances/k8sworker/scripts/setup.template.sh +++ b/instances/k8sworker/scripts/setup.template.sh @@ -33,7 +33,11 @@ fi BROADCOM_DRIVER=$(lsmod | grep bnxt_en | awk '{print $1}') if [[ -n "$${BROADCOM_DRIVER}" ]]; then echo "Disabling hardware TX checksum offloading" - ethtool --offload $(ip -o -4 route show to default | awk '{print $5}') tx off + mv /root/services/ethtool-disable-offload@.service /etc/systemd/system + systemctl daemon-reload + DEV=$(ip -o -4 route show to default | awk '{print $5}') + systemctl enable ethtool-disable-offload@$DEV + systemctl start ethtool-disable-offload@$DEV fi ## Setup NVMe drives and mount at /var/lib/docker