[PLT-225] Crossplane integration and external-dns installation from cloud-provisioner

go.mod

@@ -26,6 +26,9 @@ require (
 	golang.org/x/term v0.17.0
 	google.golang.org/api v0.149.0
 	gopkg.in/yaml.v3 v3.0.1
+	k8s.io/api v0.26.3 // indirect
+	k8s.io/apimachinery v0.26.3
+	k8s.io/client-go v0.26.1
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -39,49 +42,56 @@ require (
 )
 
 require (
-	github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect
-	github.com/cilium/ebpf v0.7.0 // indirect
-	github.com/containerd/console v1.0.3 // indirect
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
 	github.com/containers/ocicrypt v1.1.9 // indirect
 	github.com/containers/storage v1.51.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker v24.0.7+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.8.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/moby/sys/mountinfo v0.7.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/mrunalp/fileutils v0.5.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/opencontainers/runc v1.1.12 // indirect
 	github.com/opencontainers/runtime-spec v1.1.0 // indirect
-	github.com/opencontainers/selinux v1.11.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/seccomp/libseccomp-golang v0.10.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/urfave/cli v1.22.12 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
-	github.com/vishvananda/netlink v1.2.1-beta.2 // indirect
-	github.com/vishvananda/netns v0.0.4 // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	k8s.io/klog/v2 v2.80.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
 
 require (

pkg/cluster/internal/create/actions/createworker/aws.go

@@ -20,13 +20,19 @@ import (
 	"context"
 	_ "embed"
 	"encoding/base64"
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	"github.com/aws/aws-sdk-go-v2/service/ecr"
 	"gopkg.in/yaml.v3"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/kind/pkg/cluster/nodes"
 	"sigs.k8s.io/kind/pkg/commons"
 	"sigs.k8s.io/kind/pkg/errors"
@@ -39,22 +45,43 @@ var awsInternalIngress []byte
 //go:embed files/aws/public-ingress-nginx.yaml
 var awsPublicIngress []byte
 
+//go:embed files/aws/compositeresourcedefinition-hostedzones-aws.yaml
+var awsCRDHostedZones []byte
+
+//go:embed files/aws/compositeresourcedefinition-hostedzones-eks.yaml
+var eksCRDHostedZones []byte
+
 type AWSBuilder struct {
-	capxProvider     string
-	capxVersion      string
-	capxImageVersion string
-	capxManaged      bool
-	capxName         string
-	capxEnvVars      []string
-	scParameters     commons.SCParameters
-	scProvisioner    string
-	csiNamespace     string
+	capxProvider        string
+	capxVersion         string
+	capxImageVersion    string
+	capxManaged         bool
+	capxName            string
+	capxEnvVars         []string
+	scParameters        commons.SCParameters
+	scProvisioner       string
+	csiNamespace        string
+	crossplaneProviders map[string]string
 }
 
 func newAWSBuilder() *AWSBuilder {
 	return &AWSBuilder{}
 }
 
+type CrossplaneAwsParams struct {
+	Region            string
+	VPCId             string
+	ClusterName       string
+	ExternalDomain    string
+	CreateCredentials bool
+	Addon             string
+	AccountID         string
+	OIDCProviderID    string
+}
+
+var crossplaneAwsAddons = []string{"external-dns"}
+var crossplaneEKSAddons = []string{"external-dns"}
+
 func (b *AWSBuilder) setCapx(managed bool) {
 	b.capxProvider = "aws"
 	b.capxVersion = "v2.2.1"
@@ -100,6 +127,26 @@ func (b *AWSBuilder) setSC(p ProviderParams) {
 	}
 }
 
+func (b *AWSBuilder) setCrossplaneProviders(addons []string) {
+
+	b.crossplaneProviders = map[string]string{
+		"provider-family-aws": "v1.8.0",
+	}
+
+	for _, addon := range addons {
+		switch addon {
+		case "external-dns":
+			b.crossplaneProviders["provider-aws-route53"] = "v1.8.0"
+			b.crossplaneProviders["provider-aws-iam"] = "v1.8.0"
+		}
+	}
+}
+
+func (b *AWSBuilder) getCrossplaneProviders(addons []string) map[string]string {
+	b.setCrossplaneProviders(addons)
+	return b.crossplaneProviders
+}
+
 func (b *AWSBuilder) getProvider() Provider {
 	return Provider{
 		capxProvider:     b.capxProvider,
@@ -370,3 +417,204 @@ func (b *AWSBuilder) postInstallPhase(n nodes.Node, k string) error {
 
 	return nil
 }
+
+func (b *AWSBuilder) getCrossplaneProviderConfigContent(credentials map[string]*map[string]string, addon string, clusterName string, kubeconfigString string) (string, bool, error) {
+	credentialsFound := true
+	addonCredentials := credentials[addon]
+	if isEmptyCredsMap(*addonCredentials, b.capxProvider) {
+		credentialsFound = false
+		addonCredentials = credentials["crossplane"]
+	}
+	awsCredentials := "[default]\naws_access_key_id = " + (*addonCredentials)["AccessKey"] + "\naws_secret_access_key = " + (*addonCredentials)["SecretKey"] + "\n"
+	return awsCredentials, credentialsFound, nil
+}
+
+func (b *AWSBuilder) getAddons(clusterManaged bool, addonsParams map[string]*bool) []string {
+	var addons []string
+	addonsReference := crossplaneEKSAddons
+	if !clusterManaged {
+		addonsReference = crossplaneAwsAddons
+	}
+	for _, addon := range addonsReference {
+		enabled := addonsParams[addon]
+		if (enabled != nil && *enabled) || enabled == nil {
+			addons = append(addons, addon)
+		}
+	}
+
+	return addons
+}
+
+func (b *AWSBuilder) getCrossplaneCRManifests(keosCluster commons.KeosCluster, credentials map[string]string, workloadClusterInstallation bool, credentialsFound bool, addon string, customParams map[string]string) ([]string, map[string]string, error) {
+	var manifests = []string{}
+	compositionsToWait := make(map[string]string)
+	var err error = nil
+	params := CrossplaneAwsParams{
+		Region:            keosCluster.Spec.Region,
+		ClusterName:       keosCluster.Metadata.Name,
+		ExternalDomain:    keosCluster.Spec.ExternalDomain,
+		CreateCredentials: !credentialsFound,
+		Addon:             addon,
+		AccountID:         credentials["AccountID"],
+	}
+
+	switch addon {
+	case "external-dns":
+		vpcId := keosCluster.Spec.Networks.VPCID
+
+		if vpcId == "" {
+			vpcId, err = getVpcId(keosCluster, credentials)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+
+		params.VPCId = vpcId
+		if !keosCluster.Spec.ControlPlane.Managed {
+			manifests = append(manifests, string(awsCRDHostedZones))
+			compositionsToWait["xAWSZonesConfig"] = keosCluster.Metadata.Name + "-zones-config"
+			compositionHostedZones, err := getManifest("aws", "composition-hostedzones-aws.tmpl", params)
+			if err != nil {
+				return nil, nil, err
+			}
+			manifests = append(manifests, compositionHostedZones)
+			hostedZone, err := getManifest("aws", "hostedzone.aws.tmpl", params)
+			if err != nil {
+				return nil, nil, err
+			}
+			manifests = append(manifests, hostedZone)
+		} else {
+			params.OIDCProviderID = customParams["oidcProviderId"]
+			manifests = append(manifests, string(eksCRDHostedZones))
+			compositionsToWait["xAWSZonesConfig"] = keosCluster.Metadata.Name + "-zones-config"
+			compositionHostedZones, err := getManifest("aws", "composition-hostedzones-eks.tmpl", params)
+			if err != nil {
+				return nil, nil, err
+			}
+			manifests = append(manifests, compositionHostedZones)
+			hostedZone, err := getManifest("aws", "hostedzone.eks.tmpl", params)
+			if err != nil {
+				return nil, nil, err
+			}
+			manifests = append(manifests, hostedZone)
+		}
+
+	}
+
+	return manifests, compositionsToWait, nil
+}
+
+func getAWSVPCByName(config aws.Config, vpcName string) ([]string, error) {
+	vpcs := []string{}
+
+	client := ec2.NewFromConfig(config)
+	DescribeVpcOpts := &ec2.DescribeVpcsInput{Filters: []types.Filter{
+		{
+			Name:   aws.String("tag:Name"),
+			Values: []string{vpcName},
+		},
+	}}
+	output, err := client.DescribeVpcs(context.Background(), DescribeVpcOpts)
+	if err != nil {
+		return []string{}, err
+	}
+	for _, vpc := range output.Vpcs {
+		vpcs = append(vpcs, *vpc.VpcId)
+	}
+	return vpcs, nil
+}
+
+func (b *AWSBuilder) getExternalDNSCreds(n nodes.Node, clusterName string, clientset *kubernetes.Clientset, credentials map[string]string) (map[string]string, error) {
+
+	secret, err := clientset.CoreV1().Secrets("crossplane-system").Get(context.TODO(), clusterName+"-external-dns-accesskey-secret", metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get external-dns credentials secret")
+	}
+	accessKey := string(secret.Data["username"])
+	secretKey := string(secret.Data["password"])
+	externalDnsCredsMap := map[string]string{
+		"AccessKey": accessKey,
+		"SecretKey": secretKey,
+	}
+	return externalDnsCredsMap, nil
+}
+
+func (b *AWSBuilder) getAddonsReleaseInstallation(addon string) []InstallationReleases {
+	switch addon {
+	case "external-dns":
+		return []InstallationReleases{{Provider: "aws", Releases: []string{"external-dns"}}}
+	}
+	return []InstallationReleases{}
+}
+
+func (b *AWSBuilder) createExternalDNSCredsSecret(n nodes.Node, kubeconfigPath string, credentials map[string]string, managed bool, clusterName string) error {
+	if !managed {
+		c := "echo '[default]\naws_access_key_id = " + credentials["AccessKey"] + "\naws_secret_access_key = " + credentials["SecretKey"] + "\n' > " + externalDnsWorkloadCredsFile
+		// Create secret for AWS credentials
+		_, err := commons.ExecuteCommand(n, c, 3, 5)
+		if err != nil {
+			return errors.Wrap(err, "failed to create external-dns credentials secret")
+		}
+
+		c = "kubectl --kubeconfig " + kubeconfigPath + " -n external-dns create secret generic external-dns-creds" +
+			" --from-file=credentials=" + externalDnsWorkloadCredsFile
+		_, err = commons.ExecuteCommand(n, c, 3, 5)
+		if err != nil {
+			return errors.Wrap(err, "failed to create external-dns-creds credentials secret")
+		}
+	}
+	return nil
+}
+
+func getRoleArn(clusterName string, kubeconfigString string) (string, error) {
+	gvr := schema.GroupVersionResource{
+		Group:    "configs.stratio.io",
+		Version:  "v1alpha1",
+		Resource: "xawszonesconfigs",
+	}
+	xAWSZonesConfig, err := getObject(clusterName+"-zones-config", kubeconfigString, gvr, false, "")
+	if err != nil {
+		return "", err
+	}
+	roleArn := xAWSZonesConfig["status"].(map[string]interface{})["role"].(map[string]interface{})["arn"].(string)
+	if roleArn != "" {
+		return roleArn, nil
+	}
+	return "", errors.New("Role ARN not found")
+}
+
+func getVpcId(keosCluster commons.KeosCluster, credentials map[string]string) (string, error) {
+	var ctx = context.TODO()
+	cfg, err := commons.AWSGetConfig(ctx, credentials, keosCluster.Spec.Region)
+	if err != nil {
+		return "", err
+	}
+	vpcs, _ := getAWSVPCByName(cfg, keosCluster.Metadata.Name+"-vpc")
+	if len(vpcs) == 0 {
+		return "", errors.New("Cannot create Crossplane Resources: No VPCs found")
+	}
+	if len(vpcs) > 1 {
+		return "", errors.New("Cannot create Crossplane Resources: More than one VPC found")
+	}
+	return vpcs[0], nil
+}
+
+func getOIDCProviderId(clusterName string) (string, error) {
+	gvr := schema.GroupVersionResource{
+		Group:    "cluster.x-k8s.io",
+		Version:  "v1beta1",
+		Resource: "clusters",
+	}
+	cluster, err := getObject(clusterName, "", gvr, true, "cluster-"+clusterName)
+	if err != nil {
+		return "", err
+	}
+	fmt.Println("cluster")
+	fmt.Println(cluster)
+	controlplaneHost := cluster["spec"].(map[string]interface{})["controlPlaneEndpoint"].(map[string]interface{})["host"].(string)
+	if controlplaneHost == "" {
+		return "", errors.New("oidcProviderId cannot be found")
+	}
+	oidcProviderId := strings.Split(strings.Split(controlplaneHost, "//")[1], ".")[0]
+	return oidcProviderId, nil
+}