Skip to content

Commit

Permalink
helm: remove konnectivity from control-planes
Browse files Browse the repository at this point in the history 
This is the first step in our migration off of
konnectivity. Before node-to-node encryption
we used konnectivity to route some KubeAPI
to kubelet traffic over the pod network which then
would be encrypted.

Since we enabled node-to-node encryption this has no
security upsides anymore. Note that we still deploy
the konnectivity agents via helm and still have the
load balancer for konnectivity.

In the following releases we will remove both.
  • Loading branch information
@3u13r
3u13r committed Nov 15, 2023
1 parent 79f5623 commit cfcc089
Show file tree
Hide file tree
Showing 10 changed files with 23 additions and 374 deletions.
1 change: 0 additions & 1 deletion bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ go_library(
"//internal/installer",
"//internal/kubernetes",
"//internal/logger",
"//internal/role",
"//internal/versions/components",
"@com_github_coreos_go_systemd_v22//dbus",
"@com_github_spf13_afero//:afero",
Expand Down
123 changes: 2 additions & 121 deletions bootstrapper/internal/kubernetes/k8sapi/k8sutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,11 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"

"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiserver/pkg/authentication/user"
Expand Down Expand Up @@ -90,7 +87,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, kubernetesCompon
// InitCluster instruments kubeadm to initialize the K8s cluster.
// On success an admin kubeconfig file is returned.
func (k *KubernetesUtil) InitCluster(
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger,
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *logger.Logger,
) ([]byte, error) {
// TODO(3u13r): audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
Expand Down Expand Up @@ -146,12 +143,6 @@ func (k *KubernetesUtil) InitCluster(
return nil, fmt.Errorf("creating static pods directory: %w", err)
}

log.Infof("Preparing node for Konnectivity")
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return nil, fmt.Errorf("setup konnectivity: %w", err)
}

// initialize the cluster
log.Infof("Initializing the cluster using kubeadm init")
skipPhases := "--skip-phases=preflight,certs"
Expand Down Expand Up @@ -189,56 +180,6 @@ func (k *KubernetesUtil) InitCluster(
return out, nil
}

func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context, loadBalancerEndpoint string) error {
if !strings.Contains(loadBalancerEndpoint, ":") {
loadBalancerEndpoint = net.JoinHostPort(loadBalancerEndpoint, strconv.Itoa(constants.KubernetesPort))
}

konnectivityServerYaml, err := resources.NewKonnectivityServerStaticPod().Marshal()
if err != nil {
return fmt.Errorf("generating konnectivity server static pod: %w", err)
}
if err := os.WriteFile("/etc/kubernetes/manifests/konnectivity-server.yaml", konnectivityServerYaml, 0o644); err != nil {
return fmt.Errorf("writing konnectivity server pod: %w", err)
}

egressConfigYaml, err := resources.NewEgressSelectorConfiguration().Marshal()
if err != nil {
return fmt.Errorf("generating egress selector configuration: %w", err)
}
if err := os.WriteFile("/etc/kubernetes/egress-selector-configuration.yaml", egressConfigYaml, 0o644); err != nil {
return fmt.Errorf("writing egress selector config: %w", err)
}

if err := k.createSignedKonnectivityCert(); err != nil {
return fmt.Errorf("generating konnectivity server certificate: %w", err)
}

if out, err := exec.CommandContext(ctx, constants.KubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server",
"--client-certificate", "/etc/kubernetes/konnectivity.crt", "--client-key", "/etc/kubernetes/konnectivity.key", "--embed-certs=true").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-credentials: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint,
"--certificate-authority", "/etc/kubernetes/pki/ca.crt", "--embed-certs=true").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-cluster: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes",
"--cluster", "kubernetes", "--user", "system:konnectivity-server").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-context: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig use-context: %w, %s", err, string(out))
}
// cleanup
if err := os.Remove("/etc/kubernetes/konnectivity.crt"); err != nil {
return fmt.Errorf("removing konnectivity certificate: %w", err)
}
if err := os.Remove("/etc/kubernetes/konnectivity.key"); err != nil {
return fmt.Errorf("removing konnectivity key: %w", err)
}
return nil
}

// SetupPodNetworkInput holds all configuration options to setup the pod network.
type SetupPodNetworkInput struct {
CloudProvider string
Expand All @@ -250,7 +191,7 @@ type SetupPodNetworkInput struct {
}

// JoinCluster joins existing Kubernetes cluster using kubeadm join.
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error {
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error {
// TODO(3u13r): audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
if err != nil {
Expand All @@ -275,14 +216,6 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, pee
return fmt.Errorf("creating static pods directory: %w", err)
}

if peerRole == role.ControlPlane {
log.Infof("Prep Init Kubernetes cluster")
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return fmt.Errorf("setup konnectivity: %w", err)
}
}

// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
cmd := exec.CommandContext(ctx, constants.KubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
out, err := cmd.CombinedOutput()
Expand Down Expand Up @@ -369,58 +302,6 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
return k.file.Write(certificate.CertificateFilename, kubeletCert, file.OptMkdirAll)
}

// createSignedKonnectivityCert manually creates a Kubernetes CA signed certificate for the Konnectivity server.
func (k *KubernetesUtil) createSignedKonnectivityCert() error {
// Create CSR
certRequestRaw, keyPem, err := resources.GetKonnectivityCertificateRequest()
if err != nil {
return err
}
if err := k.file.Write(resources.KonnectivityKeyFilename, keyPem, file.OptMkdirAll); err != nil {
return err
}

certRequest, err := x509.ParseCertificateRequest(certRequestRaw)
if err != nil {
return err
}

// Prepare certificate signing
serialNumber, err := crypto.GenerateCertificateSerialNumber()
if err != nil {
return err
}

now := time.Now()
// Create the kubelet certificate
// For a reference on the certificate fields, see: https://kubernetes.io/docs/setup/best-practices/certificates/
certTmpl := &x509.Certificate{
SerialNumber: serialNumber,
NotBefore: now.Add(-2 * time.Hour),
NotAfter: now.Add(24 * 365 * time.Hour),
Subject: certRequest.Subject,
}

parentCert, parentKey, err := k.getKubernetesCACertAndKey()
if err != nil {
return err
}

// Sign the certificate
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, certRequest.PublicKey, parentKey)
if err != nil {
return err
}

// Write the certificate
konnectivityCert := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: certRaw,
})

return k.file.Write(resources.KonnectivityCertificateFilename, konnectivityCert, file.OptMkdirAll)
}

// getKubernetesCACertAndKey returns the Kubernetes CA certificate and key.
// The key of type `any` can be consumed by `x509.CreateCertificate()`.
func (k *KubernetesUtil) getKubernetesCACertAndKey() (*x509.Certificate, any, error) {
Expand Down
21 changes: 3 additions & 18 deletions bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,9 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
"audit-log-path": filepath.Join(auditLogDir, auditLogFile), // CIS benchmark
"audit-log-maxage": "30", // CIS benchmark - Default value of Rancher
// log size = 10 files * 100MB + 100 MB (which is currently being written) = 1.1GB
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
"profiling": "false", // CIS benchmark
"egress-selector-config-file": "/etc/kubernetes/egress-selector-configuration.yaml",
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
"profiling": "false", // CIS benchmark
"kubelet-certificate-authority": filepath.Join(
kubeconstants.KubernetesDir,
kubeconstants.DefaultCertificateDir,
Expand Down Expand Up @@ -104,20 +103,6 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
ReadOnly: true,
PathType: corev1.HostPathFile,
},
{
Name: "egress-config",
HostPath: "/etc/kubernetes/egress-selector-configuration.yaml",
MountPath: "/etc/kubernetes/egress-selector-configuration.yaml",
ReadOnly: true,
PathType: corev1.HostPathFile,
},
{
Name: "konnectivity-uds",
HostPath: "/run/konnectivity-server",
MountPath: "/run/konnectivity-server",
ReadOnly: false,
PathType: corev1.HostPathDirectoryOrCreate,
},
},
},
CertSANs: []string{"127.0.0.1"},
Expand Down
6 changes: 0 additions & 6 deletions bootstrapper/internal/kubernetes/k8sapi/resources/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,13 @@ go_library(
name = "resources",
srcs = [
"auditpolicy.go",
"konnectivity.go",
"resources.go",
],
importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources",
visibility = ["//bootstrapper:__subpackages__"],
deps = [
"//bootstrapper/internal/certificate",
"//internal/kubernetes",
"//internal/versions",
"@io_k8s_api//core/v1:core",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/util/intstr",
"@io_k8s_apiserver//pkg/apis/apiserver",
"@io_k8s_apiserver//pkg/apis/audit/v1:audit",
],
)
Expand Down
Loading

0 comments on commit cfcc089

Please sign in to comment.