Skip to content

Commit

Permalink
remove konnectivity
Browse files Browse the repository at this point in the history
  • Loading branch information
@3u13r
3u13r committed Oct 23, 2023
1 parent 2b4609e commit cbc7406
Show file tree
Hide file tree
Showing 42 changed files with 20 additions and 1,068 deletions.
1 change: 0 additions & 1 deletion bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ go_library(
"//internal/installer",
"//internal/kubernetes",
"//internal/logger",
"//internal/role",
"//internal/versions/components",
"@com_github_coreos_go_systemd_v22//dbus",
"@com_github_spf13_afero//:afero",
Expand Down
122 changes: 2 additions & 120 deletions bootstrapper/internal/kubernetes/k8sapi/k8sutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,12 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"

"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiserver/pkg/authentication/user"
Expand Down Expand Up @@ -91,7 +89,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, kubernetesCompon
// InitCluster instruments kubeadm to initialize the K8s cluster.
// On success an admin kubeconfig file is returned.
func (k *KubernetesUtil) InitCluster(
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, controlPlaneHost, controlPlanePort string, conformanceMode bool, log *logger.Logger,
ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *logger.Logger,
) ([]byte, error) {
// TODO(3u13r): audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
Expand Down Expand Up @@ -147,12 +145,6 @@ func (k *KubernetesUtil) InitCluster(
return nil, fmt.Errorf("creating static pods directory: %w", err)
}

log.Infof("Preparing node for Konnectivity")
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return nil, fmt.Errorf("setup konnectivity: %w", err)
}

// initialize the cluster
log.Infof("Initializing the cluster using kubeadm init")
skipPhases := "--skip-phases=preflight,certs"
Expand Down Expand Up @@ -190,56 +182,6 @@ func (k *KubernetesUtil) InitCluster(
return out, nil
}

func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context, loadBalancerEndpoint string) error {
if !strings.Contains(loadBalancerEndpoint, ":") {
loadBalancerEndpoint = net.JoinHostPort(loadBalancerEndpoint, strconv.Itoa(constants.KubernetesPort))
}

konnectivityServerYaml, err := resources.NewKonnectivityServerStaticPod().Marshal()
if err != nil {
return fmt.Errorf("generating konnectivity server static pod: %w", err)
}
if err := os.WriteFile("/etc/kubernetes/manifests/konnectivity-server.yaml", konnectivityServerYaml, 0o644); err != nil {
return fmt.Errorf("writing konnectivity server pod: %w", err)
}

egressConfigYaml, err := resources.NewEgressSelectorConfiguration().Marshal()
if err != nil {
return fmt.Errorf("generating egress selector configuration: %w", err)
}
if err := os.WriteFile("/etc/kubernetes/egress-selector-configuration.yaml", egressConfigYaml, 0o644); err != nil {
return fmt.Errorf("writing egress selector config: %w", err)
}

if err := k.createSignedKonnectivityCert(); err != nil {
return fmt.Errorf("generating konnectivity server certificate: %w", err)
}

if out, err := exec.CommandContext(ctx, constants.KubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server",
"--client-certificate", "/etc/kubernetes/konnectivity.crt", "--client-key", "/etc/kubernetes/konnectivity.key", "--embed-certs=true").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-credentials: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint,
"--certificate-authority", "/etc/kubernetes/pki/ca.crt", "--embed-certs=true").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-cluster: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes",
"--cluster", "kubernetes", "--user", "system:konnectivity-server").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig set-context: %w, %s", err, string(out))
}
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil {
return fmt.Errorf("konnectivity kubeconfig use-context: %w, %s", err, string(out))
}
// cleanup
if err := os.Remove("/etc/kubernetes/konnectivity.crt"); err != nil {
return fmt.Errorf("removing konnectivity certificate: %w", err)
}
if err := os.Remove("/etc/kubernetes/konnectivity.key"); err != nil {
return fmt.Errorf("removing konnectivity key: %w", err)
}
return nil
}

// SetupPodNetworkInput holds all configuration options to setup the pod network.
type SetupPodNetworkInput struct {
CloudProvider string
Expand Down Expand Up @@ -316,7 +258,7 @@ func (k *KubernetesUtil) FixCilium(ctx context.Context) error {
}

// JoinCluster joins existing Kubernetes cluster using kubeadm join.
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneHost, controlPlanePort string, log *logger.Logger) error {
func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error {
// TODO(3u13r): audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
if err != nil {
Expand All @@ -341,14 +283,6 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, pee
return fmt.Errorf("creating static pods directory: %w", err)
}

if peerRole == role.ControlPlane {
log.Infof("Prep Init Kubernetes cluster")
controlPlaneEndpoint := net.JoinHostPort(controlPlaneHost, controlPlanePort)
if err := k.prepareControlPlaneForKonnectivity(ctx, controlPlaneEndpoint); err != nil {
return fmt.Errorf("setup konnectivity: %w", err)
}
}

// run `kubeadm join` to join a worker node to an existing Kubernetes cluster
cmd := exec.CommandContext(ctx, constants.KubeadmPath, "join", "-v=5", "--config", joinConfigFile.Name())
out, err := cmd.CombinedOutput()
Expand Down Expand Up @@ -435,58 +369,6 @@ func (k *KubernetesUtil) createSignedKubeletCert(nodeName string, ips []net.IP)
return k.file.Write(certificate.CertificateFilename, kubeletCert, file.OptMkdirAll)
}

// createSignedKonnectivityCert manually creates a Kubernetes CA signed certificate for the Konnectivity server.
func (k *KubernetesUtil) createSignedKonnectivityCert() error {
// Create CSR
certRequestRaw, keyPem, err := resources.GetKonnectivityCertificateRequest()
if err != nil {
return err
}
if err := k.file.Write(resources.KonnectivityKeyFilename, keyPem, file.OptMkdirAll); err != nil {
return err
}

certRequest, err := x509.ParseCertificateRequest(certRequestRaw)
if err != nil {
return err
}

// Prepare certificate signing
serialNumber, err := crypto.GenerateCertificateSerialNumber()
if err != nil {
return err
}

now := time.Now()
// Create the kubelet certificate
// For a reference on the certificate fields, see: https://kubernetes.io/docs/setup/best-practices/certificates/
certTmpl := &x509.Certificate{
SerialNumber: serialNumber,
NotBefore: now.Add(-2 * time.Hour),
NotAfter: now.Add(24 * 365 * time.Hour),
Subject: certRequest.Subject,
}

parentCert, parentKey, err := k.getKubernetesCACertAndKey()
if err != nil {
return err
}

// Sign the certificate
certRaw, err := x509.CreateCertificate(rand.Reader, certTmpl, parentCert, certRequest.PublicKey, parentKey)
if err != nil {
return err
}

// Write the certificate
konnectivityCert := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: certRaw,
})

return k.file.Write(resources.KonnectivityCertificateFilename, konnectivityCert, file.OptMkdirAll)
}

// getKubernetesCACertAndKey returns the Kubernetes CA certificate and key.
// The key of type `any` can be consumed by `x509.CreateCertificate()`.
func (k *KubernetesUtil) getKubernetesCACertAndKey() (*x509.Certificate, any, error) {
Expand Down
14 changes: 3 additions & 11 deletions bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,9 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
"audit-log-path": filepath.Join(auditLogDir, auditLogFile), // CIS benchmark
"audit-log-maxage": "30", // CIS benchmark - Default value of Rancher
// log size = 10 files * 100MB + 100 MB (which is currently being written) = 1.1GB
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
"profiling": "false", // CIS benchmark
"egress-selector-config-file": "/etc/kubernetes/egress-selector-configuration.yaml",
"audit-log-maxbackup": "10", // CIS benchmark - Default value of Rancher
"audit-log-maxsize": "100", // CIS benchmark - Default value of Rancher
"profiling": "false", // CIS benchmark
"kubelet-certificate-authority": filepath.Join(
kubeconstants.KubernetesDir,
kubeconstants.DefaultCertificateDir,
Expand Down Expand Up @@ -111,13 +110,6 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
ReadOnly: true,
PathType: corev1.HostPathFile,
},
{
Name: "konnectivity-uds",
HostPath: "/run/konnectivity-server",
MountPath: "/run/konnectivity-server",
ReadOnly: false,
PathType: corev1.HostPathDirectoryOrCreate,
},
},
},
CertSANs: []string{"127.0.0.1"},
Expand Down
6 changes: 0 additions & 6 deletions bootstrapper/internal/kubernetes/k8sapi/resources/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,13 @@ go_library(
name = "resources",
srcs = [
"auditpolicy.go",
"konnectivity.go",
"resources.go",
],
importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources",
visibility = ["//bootstrapper:__subpackages__"],
deps = [
"//bootstrapper/internal/certificate",
"//internal/kubernetes",
"//internal/versions",
"@io_k8s_api//core/v1:core",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/util/intstr",
"@io_k8s_apiserver//pkg/apis/apiserver",
"@io_k8s_apiserver//pkg/apis/audit/v1:audit",
],
)
Expand Down
Loading

0 comments on commit cbc7406

Please sign in to comment.