diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 029068c1d2cf3..9921688291f40 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -94,7 +94,7 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error { } authenticator = a - case "metal": + case kops.CloudProviderMetal: a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem") if err != nil { return err diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index 1bae36ca3e6e5..c81cfd3436857 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -112,6 +112,8 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(cluster *kops.Cluster) error c.CloudProvider = "azure" case kops.CloudProviderScaleway: c.CloudProvider = "external" + case kops.CloudProviderMetal: + c.CloudProvider = "external" default: return fmt.Errorf("unknown cloudprovider %q", cluster.GetCloudProvider()) } diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index bd8c6f606dab7..68d8961d5dafc 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -525,6 +525,11 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance fmt.Sprintf("%s=%s", scaleway.TagNameRolePrefix, scaleway.TagRoleControlPlane), } config.VolumeNameTag = fmt.Sprintf("%s=%s", scaleway.TagInstanceGroup, instanceGroupName) + + case kops.CloudProviderMetal: + config.VolumeProvider = "external" + // TODO: Use static configuration here? + default: return nil, fmt.Errorf("CloudProvider %q not supported with etcd-manager", b.Cluster.GetCloudProvider()) } diff --git a/pkg/model/master_volumes.go b/pkg/model/master_volumes.go index cde9d27de6f71..3f3d734cad7cc 100644 --- a/pkg/model/master_volumes.go +++ b/pkg/model/master_volumes.go @@ -122,6 +122,10 @@ func (b *MasterVolumeBuilder) Build(c *fi.CloudupModelBuilderContext) error { } case kops.CloudProviderScaleway: b.addScalewayVolume(c, name, volumeSize, zone, etcd, m, allMembers) + + case kops.CloudProviderMetal: + // Nothing special to do for Metal (yet) + default: return fmt.Errorf("unknown cloudprovider %q", b.Cluster.GetCloudProvider()) } diff --git a/tests/e2e/scenarios/bare-metal/run-test b/tests/e2e/scenarios/bare-metal/run-test index db26f675fc64a..67f147423f904 100755 --- a/tests/e2e/scenarios/bare-metal/run-test +++ b/tests/e2e/scenarios/bare-metal/run-test @@ -61,7 +61,17 @@ export S3_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} # Create the state-store bucket in our mock s3 server export KOPS_STATE_STORE=s3://kops-state-store/ aws --version -aws --endpoint-url=${S3_ENDPOINT} --debug s3 mb s3://kops-state-store +aws --endpoint-url=${S3_ENDPOINT} s3 mb s3://kops-state-store # List clusters (there should not be any yet) go run ./cmd/kops get cluster || true + +# Create a cluster +go run ./cmd/kops create cluster --cloud=metal metal.k8s.local --zones main + +# List clusters +go run ./cmd/kops get cluster + +# List instance groups +go run ./cmd/kops get ig --name metal.k8s.local +go run ./cmd/kops get ig --name metal.k8s.local -oyaml diff --git a/tools/metal/storage/main.go b/tools/metal/storage/main.go index 50a44b1680024..59316000a0009 100644 --- a/tools/metal/storage/main.go +++ b/tools/metal/storage/main.go @@ -22,9 +22,9 @@ import ( "flag" "fmt" "net/http" + "net/url" "os" "strings" - "time" "github.com/kubernetes/kops/tools/metal/dhcp/pkg/objectstore" "github.com/kubernetes/kops/tools/metal/dhcp/pkg/objectstore/testobjectstore" @@ -90,7 +90,7 @@ func (s *S3Server) ListAllMyBuckets(ctx context.Context, req *s3Request, r *List for _, bucket := range s.store.ListBuckets(ctx) { output.Buckets = append(output.Buckets, s3model.Bucket{ - CreationDate: bucket.CreationDate.Format(time.RFC3339), + CreationDate: bucket.CreationDate.Format(s3TimeFormat), Name: bucket.Name, }) } @@ -107,6 +107,11 @@ func (s *S3Server) ServeRequest(ctx context.Context, w http.ResponseWriter, r *h tokens := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") + values, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return fmt.Errorf("failed to parse query: %w", err) + } + req := &s3Request{ w: w, r: r, @@ -121,7 +126,9 @@ func (s *S3Server) ServeRequest(ctx context.Context, w http.ResponseWriter, r *h switch r.Method { case http.MethodGet: return s.ListObjectsV2(ctx, req, &ListObjectsV2Input{ - Bucket: bucket, + Bucket: bucket, + Delimiter: values.Get("delimiter"), + Prefix: values.Get("prefix"), }) case http.MethodPut: return s.CreateBucket(ctx, req, &CreateBucketInput{ @@ -136,10 +143,22 @@ func (s *S3Server) ServeRequest(ctx context.Context, w http.ResponseWriter, r *h if len(tokens) > 1 { bucket := tokens[0] - return s.GetObject(ctx, req, &GetObjectInput{ - Bucket: bucket, - Key: strings.TrimPrefix(r.URL.Path, "/"+bucket+"/"), - }) + key := strings.TrimPrefix(r.URL.Path, "/"+bucket+"/") + switch r.Method { + case http.MethodGet: + return s.GetObject(ctx, req, &GetObjectInput{ + Bucket: bucket, + Key: key, + }) + case http.MethodPut: + return s.PutObject(ctx, req, &PutObjectInput{ + Bucket: bucket, + Key: key, + }) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return nil + } } return fmt.Errorf("unhandled path %q", r.URL.Path) @@ -147,8 +166,13 @@ func (s *S3Server) ServeRequest(ctx context.Context, w http.ResponseWriter, r *h type ListObjectsV2Input struct { Bucket string + + Delimiter string + Prefix string } +const s3TimeFormat = "2006-01-02T15:04:05.000Z" + func (s *S3Server) ListObjectsV2(ctx context.Context, req *s3Request, input *ListObjectsV2Input) error { bucket, err := s.store.GetBucket(ctx, input.Bucket) if err != nil { @@ -168,12 +192,19 @@ func (s *S3Server) ListObjectsV2(ctx context.Context, req *s3Request, input *Lis } for _, object := range objects { + if input.Prefix != "" && !strings.HasPrefix(object.Key, input.Prefix) { + continue + } + // if input.Delimiter != "" && strings.Contains(object.Key, input.Delimiter) { + // continue + // } output.Contents = append(output.Contents, s3model.Object{ Key: object.Key, - LastModified: object.LastModified.Format(time.RFC3339), + LastModified: object.LastModified.Format(s3TimeFormat), Size: object.Size, }) } + output.KeyCount = len(output.Contents) return req.writeXML(ctx, output) } @@ -232,6 +263,31 @@ func (s *S3Server) GetObject(ctx context.Context, req *s3Request, input *GetObje return object.WriteTo(req.w) } +type PutObjectInput struct { + Bucket string + Key string +} + +func (s *S3Server) PutObject(ctx context.Context, req *s3Request, input *PutObjectInput) error { + log := klog.FromContext(ctx) + + bucket, err := s.store.GetBucket(ctx, input.Bucket) + if err != nil { + return fmt.Errorf("failed to get bucket %q: %w", input.Bucket, err) + } + if bucket == nil { + return req.writeError(ctx, http.StatusNotFound, nil) + } + + objectInfo, err := bucket.PutObject(ctx, input.Key, req.r.Body) + if err != nil { + return fmt.Errorf("failed to create object %q in bucket %q: %w", input.Key, input.Bucket, err) + } + log.Info("object created", "object", objectInfo) + + return nil +} + type s3Request struct { Action string Version string diff --git a/tools/metal/storage/pkg/objectstore/interfaces.go b/tools/metal/storage/pkg/objectstore/interfaces.go index 3678e22cf978d..f070282f95c4b 100644 --- a/tools/metal/storage/pkg/objectstore/interfaces.go +++ b/tools/metal/storage/pkg/objectstore/interfaces.go @@ -18,6 +18,7 @@ package objectstore import ( "context" + "io" "net/http" "time" ) @@ -44,6 +45,9 @@ type Bucket interface { // If the object does not exist, it returns (nil, nil). GetObject(ctx context.Context, key string) (Object, error) + // PutObject creates the object with the given key. + PutObject(ctx context.Context, key string, r io.Reader) (*ObjectInfo, error) + // ListObjects returns the list of objects in the bucket. ListObjects(ctx context.Context) ([]ObjectInfo, error) } diff --git a/tools/metal/storage/pkg/objectstore/testobjectstore/testobjectstore.go b/tools/metal/storage/pkg/objectstore/testobjectstore/testobjectstore.go index 980064ba29245..55e04af504274 100644 --- a/tools/metal/storage/pkg/objectstore/testobjectstore/testobjectstore.go +++ b/tools/metal/storage/pkg/objectstore/testobjectstore/testobjectstore.go @@ -18,6 +18,8 @@ package testobjectstore import ( "context" + "fmt" + "io" "net/http" "sync" "time" @@ -119,6 +121,28 @@ func (m *TestBucket) GetObject(ctx context.Context, key string) (objectstore.Obj return obj, nil } +func (m *TestBucket) PutObject(ctx context.Context, key string, r io.Reader) (*objectstore.ObjectInfo, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + b, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("reading data: %w", err) + } + + info := objectstore.ObjectInfo{ + Key: key, + LastModified: time.Now().UTC(), + Size: int64(len(b)), + } + + m.objects[key] = &TestObject{ + data: b, + info: info, + } + return &info, nil +} + type TestObject struct { data []byte info objectstore.ObjectInfo diff --git a/tools/metal/storage/pkg/s3model/api.go b/tools/metal/storage/pkg/s3model/api.go index 36950bb4ddf0f..95ed2a08b93d5 100644 --- a/tools/metal/storage/pkg/s3model/api.go +++ b/tools/metal/storage/pkg/s3model/api.go @@ -49,14 +49,14 @@ type ListBucketResult struct { } type Object struct { - ChecksumAlgorithm string `xml:"ChecksumAlgorithm"` - ETag string `xml:"ETag"` - Key string `xml:"Key"` - LastModified string `xml:"LastModified"` - Owner Owner `xml:"Owner"` - RestoreStatus RestoreStatus `xml:"RestoreStatus"` - Size int64 `xml:"Size"` - StorageClass string `xml:"StorageClass"` + ChecksumAlgorithm string `xml:"ChecksumAlgorithm"` + ETag string `xml:"ETag"` + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + Owner *Owner `xml:"Owner"` + RestoreStatus *RestoreStatus `xml:"RestoreStatus"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` } type Owner struct { DisplayName string `xml:"DisplayName"` @@ -64,6 +64,6 @@ type Owner struct { } type RestoreStatus struct { - IsRestoreInProgress bool `xml:"IsRestoreInProgress"` - RestoreExpiryDate string `xml:"RestoreExpiryDate"` + IsRestoreInProgress bool `xml:"IsRestoreInProgress"` + RestoreExpiryDate *string `xml:"RestoreExpiryDate"` } diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index e0338a084f87e..5bc1f57c495fe 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -482,6 +482,9 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { scwZone = scwCloud.Zone() } + case kops.CloudProviderMetal: + // Metal is a special case, we don't need to do anything here (yet) + default: return nil, fmt.Errorf("unknown CloudProvider %q", cluster.GetCloudProvider()) } @@ -686,6 +689,9 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { &scalewaymodel.SSHKeyModelBuilder{ScwModelContext: scwModelContext, Lifecycle: securityLifecycle}, ) + case kops.CloudProviderMetal: + // No special builders for bare metal (yet) + default: return nil, fmt.Errorf("unknown cloudprovider %q", cluster.GetCloudProvider()) } diff --git a/upup/pkg/fi/cloudup/metal/api_target.go b/upup/pkg/fi/cloudup/metal/api_target.go new file mode 100644 index 0000000000000..e1a37885e3869 --- /dev/null +++ b/upup/pkg/fi/cloudup/metal/api_target.go @@ -0,0 +1,50 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metal + +import ( + "k8s.io/klog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" +) + +type APITarget struct { + Cloud *Cloud + OtherClouds []fi.Cloud +} + +var _ fi.CloudupTarget = &APITarget{} + +func NewAPITarget(cloud *Cloud, otherClouds []fi.Cloud) *APITarget { + return &APITarget{ + Cloud: cloud, + OtherClouds: otherClouds, + } +} + +func (t *APITarget) GetAWSCloud() awsup.AWSCloud { + klog.Fatalf("cannot find instance of AWSCloud in context") + return nil +} + +func (t *APITarget) Finish(taskMap map[string]fi.CloudupTask) error { + return nil +} + +func (t *APITarget) DefaultCheckExisting() bool { + return true +} diff --git a/upup/pkg/fi/cloudup/metal/cloud.go b/upup/pkg/fi/cloudup/metal/cloud.go new file mode 100644 index 0000000000000..badf41924a4b9 --- /dev/null +++ b/upup/pkg/fi/cloudup/metal/cloud.go @@ -0,0 +1,108 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metal + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "k8s.io/kops/dnsprovider/pkg/dnsprovider" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/cloudinstances" + "k8s.io/kops/upup/pkg/fi" +) + +var _ fi.Cloud = &Cloud{} + +// Cloud holds the fi.Cloud implementation for metal resources. +type Cloud struct { +} + +// NewCloud returns a Cloud for metal resources. +func NewCloud() (*Cloud, error) { + cloud := &Cloud{} + return cloud, nil +} + +func (c *Cloud) ProviderID() kops.CloudProviderID { + return kops.CloudProviderMetal +} +func (c *Cloud) DNS() (dnsprovider.Interface, error) { + return nil, fmt.Errorf("method not implemented") +} + +// FindVPCInfo looks up the specified VPC by id, returning info if found, otherwise (nil, nil). +func (c *Cloud) FindVPCInfo(id string) (*fi.VPCInfo, error) { + return nil, fmt.Errorf("method not implemented") +} + +// DeleteInstance deletes a cloud instance. +func (c *Cloud) DeleteInstance(instance *cloudinstances.CloudInstance) error { + return fmt.Errorf("method not implemented") +} + +// // DeregisterInstance drains a cloud instance and loadbalancers. +func (c *Cloud) DeregisterInstance(instance *cloudinstances.CloudInstance) error { + return fmt.Errorf("method not implemented") +} + +// DeleteGroup deletes the cloud resources that make up a CloudInstanceGroup, including the instances. +func (c *Cloud) DeleteGroup(group *cloudinstances.CloudInstanceGroup) error { + return fmt.Errorf("method not implemented") +} + +// DetachInstance causes a cloud instance to no longer be counted against the group's size limits. +func (c *Cloud) DetachInstance(instance *cloudinstances.CloudInstance) error { + return fmt.Errorf("method not implemented") +} + +// GetCloudGroups returns a map of cloud instances that back a kops cluster. +// Detached instances must be returned in the NeedUpdate slice. +func (c *Cloud) GetCloudGroups(cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, warnUnmatched bool, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) { + return nil, fmt.Errorf("method not implemented") +} + +// Region returns the cloud region bound to the cloud instance. +// If the region concept does not apply, returns "". +func (c *Cloud) Region() string { + return "" +} + +// FindClusterStatus discovers the status of the cluster, by inspecting the cloud objects +func (c *Cloud) FindClusterStatus(cluster *kops.Cluster) (*kops.ClusterStatus, error) { + // etcdStatus, err := findEtcdStatus(c, cluster) + // if err != nil { + // return nil, err + // } + klog.Warningf("method metal.Cloud::FindClusterStatus stub-implemented") + return &kops.ClusterStatus{ + // EtcdClusters: etcdStatus, + }, nil + + // return nil, fmt.Errorf("method metal.Cloud::FindClusterStatus not implemented") +} + +func (c *Cloud) GetApiIngressStatus(cluster *kops.Cluster) ([]fi.ApiIngressStatus, error) { + // name := "api." + cluster.Name + var ret []fi.ApiIngressStatus + ret = append(ret, fi.ApiIngressStatus{ + // Hostname: name, + IP: "2605:a601:55b6:2d00:7878:1:0:1", + }) + return ret, nil +} diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index bee81abcf7bb8..7a91b085835fa 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -359,7 +359,11 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster cloud = osCloud case api.CloudProviderScaleway: cluster.Spec.CloudProvider.Scaleway = &api.ScalewaySpec{} - + case api.CloudProviderMetal: + if cluster.Labels == nil { + cluster.Labels = make(map[string]string) + } + cluster.Labels[api.AlphaLabelCloudProvider] = string(api.CloudProviderMetal) default: return nil, fmt.Errorf("unsupported cloud provider %s", opt.CloudProvider) } @@ -1652,6 +1656,8 @@ func defaultImage(cluster *api.Cluster, channel *api.Channel, architecture archi return defaultHetznerImageJammy, nil case api.CloudProviderScaleway: return defaultScalewayImageJammy, nil + case api.CloudProviderMetal: + return "dummy-metal-image", nil } } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index c1faefd490176..df3dce29d1b14 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -762,6 +762,10 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { ClusterName: tf.ClusterName(), } + case kops.CloudProviderMetal: + // Use crypto public/private keys for Metal + config.Server.PKI = &pkibootstrap.Options{} + default: return "", fmt.Errorf("unsupported cloud provider %s", cluster.GetCloudProvider()) } diff --git a/upup/pkg/fi/cloudup/utils.go b/upup/pkg/fi/cloudup/utils.go index 7f45e20b21a3a..262929bd054e4 100644 --- a/upup/pkg/fi/cloudup/utils.go +++ b/upup/pkg/fi/cloudup/utils.go @@ -31,6 +31,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/do" "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" + "k8s.io/kops/upup/pkg/fi/cloudup/metal" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" ) @@ -193,6 +194,12 @@ func BuildCloud(cluster *kops.Cluster) (fi.Cloud, error) { cloud = scwCloud } + case kops.CloudProviderMetal: + metalCloud, err := metal.NewCloud() + if err != nil { + return nil, fmt.Errorf("error initializing Metal cloud: %w", err) + } + cloud = metalCloud default: return nil, fmt.Errorf("unknown CloudProvider %q", cluster.GetCloudProvider()) }