diff --git a/charts/proxmox-csi-plugin/Chart.yaml b/charts/proxmox-csi-plugin/Chart.yaml index 6861cdd..e673fc7 100644 --- a/charts/proxmox-csi-plugin/Chart.yaml +++ b/charts/proxmox-csi-plugin/Chart.yaml @@ -18,7 +18,7 @@ maintainers: url: https://github.com/sergelogvinov # # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.1 +version: 0.3.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/proxmox-csi-plugin/values.yaml b/charts/proxmox-csi-plugin/values.yaml index 5e24371..dd6baa7 100644 --- a/charts/proxmox-csi-plugin/values.yaml +++ b/charts/proxmox-csi-plugin/values.yaml @@ -51,6 +51,9 @@ configFile: /etc/proxmox/config.yaml # -- Proxmox cluster config. config: + features: + # specify provider: proxmox if you are using capmox (cluster api provider for proxmox) + provider: 'default' clusters: [] # - url: https://cluster-api-1.exmple.com:8006/api2/json # insecure: false diff --git a/go.mod b/go.mod index 925dbe7..896e989 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/jarcoal/httpmock v1.3.1 github.com/kubernetes-csi/csi-lib-utils v0.20.0 - github.com/sergelogvinov/proxmox-cloud-controller-manager v0.5.1 + github.com/sergelogvinov/proxmox-cloud-controller-manager v0.7.0 github.com/siderolabs/go-blockdevice v0.4.8 github.com/siderolabs/go-retry v0.3.3 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index db73920..3138671 100644 --- a/go.sum +++ b/go.sum @@ -102,8 +102,11 @@ github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoG github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergelogvinov/proxmox-cloud-controller-manager v0.5.1 h1:RM4D/VCreZr22pTuJHd97Rz8DrDUYMAxlv6/xf65EoA= github.com/sergelogvinov/proxmox-cloud-controller-manager v0.5.1/go.mod h1:m9Ip+Wu/kDNwsemzlQlqOMUrCmvsxA1zbI32dQHgPCw= +github.com/sergelogvinov/proxmox-cloud-controller-manager v0.6.1-0.20250107152926-956a30a46389 h1:Ui8MUcSqpPgJm9E6gnxtyA6Pf7KkDNmkwfHe0gynlyk= +github.com/sergelogvinov/proxmox-cloud-controller-manager v0.6.1-0.20250107152926-956a30a46389/go.mod h1:LePNcAZ6SA+yh5WttDmxFgs1CUJLu07tOajAzyuiKhI= +github.com/sergelogvinov/proxmox-cloud-controller-manager v0.7.0 h1:RUxT3QVyVR2nMiSwopTP0rlbqWHbYy53AyMnE1qz0so= +github.com/sergelogvinov/proxmox-cloud-controller-manager v0.7.0/go.mod h1:LePNcAZ6SA+yh5WttDmxFgs1CUJLu07tOajAzyuiKhI= github.com/siderolabs/go-blockdevice v0.4.8 h1:KfdWvIx0Jft5YVuCsFIJFwjWEF1oqtzkgX9PeU9cX4c= github.com/siderolabs/go-blockdevice v0.4.8/go.mod h1:4PeOuk71pReJj1JQEXDE7kIIQJPVe8a+HZQa+qjxSEA= github.com/siderolabs/go-cmd v0.1.3 h1:JrgZwqhJQeoec3QRON0LK+fv+0y7d0DyY7zsfkO6ciw= @@ -142,8 +145,8 @@ go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37Cb golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= +golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/pkg/csi/controller.go b/pkg/csi/controller.go index b784e05..134c76d 100644 --- a/pkg/csi/controller.go +++ b/pkg/csi/controller.go @@ -57,9 +57,9 @@ var controllerCaps = []csi.ControllerServiceCapability_RPC_Type{ // ControllerService is the controller service for the CSI driver type ControllerService struct { - Cluster *proxmox.Cluster - Kclient clientkubernetes.Interface - + Cluster *proxmox.Cluster + Kclient clientkubernetes.Interface + Provider proxmox.Provider volumeLocks sync.Mutex csi.UnimplementedControllerServer @@ -78,8 +78,9 @@ func NewControllerService(kclient *clientkubernetes.Clientset, cloudConfig strin } return &ControllerService{ - Cluster: cluster, - Kclient: kclient, + Cluster: cluster, + Kclient: kclient, + Provider: cfg.Features.Provider, }, nil } @@ -342,7 +343,8 @@ func (d *ControllerService) ControllerPublishVolume(ctx context.Context, request var vmr *pxapi.VmRef - vmrid, zone, err := tools.ProxmoxVMID(ctx, d.Kclient, nodeID) + vmrid, zone, err := tools.ProxmoxVMID(ctx, d.Kclient, cl, nodeID, d.Provider) + if err != nil { klog.InfoS("ControllerPublishVolume: failed to get proxmox vmrID from ProviderID", "cluster", vol.Cluster(), "nodeID", nodeID) @@ -466,7 +468,8 @@ func (d *ControllerService) ControllerUnpublishVolume(ctx context.Context, reque var vmr *pxapi.VmRef - vmrid, zone, err := tools.ProxmoxVMID(ctx, d.Kclient, nodeID) + vmrid, zone, err := tools.ProxmoxVMID(ctx, d.Kclient, cl, nodeID, d.Provider) + if err != nil { klog.InfoS("ControllerUnpublishVolume: failed to get proxmox vmrID from ProviderID", "cluster", vol.Cluster(), "nodeID", nodeID) diff --git a/pkg/csi/controller_test.go b/pkg/csi/controller_test.go index 86c40a2..fff5bf7 100644 --- a/pkg/csi/controller_test.go +++ b/pkg/csi/controller_test.go @@ -41,14 +41,25 @@ import ( var _ proto.ControllerServer = (*csi.ControllerService)(nil) -type csiTestSuite struct { +type baseCSITestSuite struct { suite.Suite - s *csi.ControllerService } -func (ts *csiTestSuite) SetupTest() { - cfg, err := proxmox.ReadCloudConfig(strings.NewReader(` +type configTestCase struct { + name string + config string + providerID string +} + +func getTestConfigs() []configTestCase { + return []configTestCase{ + { + name: "CapMoxProvider", + providerID: "proxmox://11833f4c-341f-4bd3-aad7-f7abeda472e6", + config: ` +features: + provider: capmox clusters: - url: https://127.0.0.1:8006/api2/json insecure: false @@ -59,12 +70,36 @@ clusters: insecure: false token_id: "user!token-id" token_secret: "secret" - region: cluster-2 -`)) - if err != nil { - ts.T().Fatalf("failed to read config: %v", err) + region: cluster-2`, + }, + { + name: "ExplicitDefaultProvider", + providerID: "proxmox://cluster-1/101", + config: ` +features: + provider: capmox +clusters: +- url: https://127.0.0.1:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-1`, + }, + { + name: "ImplicitDefaultProvider", + providerID: "proxmox://cluster-1/101", + config: ` +clusters: +- url: https://127.0.0.1:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-1`, + }, } +} +func setupMockResponders() { httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/cluster/resources", func(_ *http.Request) (*http.Response, error) { return httpmock.NewJsonResponse(200, map[string]interface{}{ @@ -286,10 +321,12 @@ clusters: }) }, ) +} - cluster, err := proxmox.NewCluster(&cfg, &http.Client{}) +func (ts *baseCSITestSuite) setupTestSuite(config string, providerID string) error { + cfg, err := proxmox.ReadCloudConfig(strings.NewReader(config)) if err != nil { - ts.T().Fatalf("failed to create proxmox cluster client: %v", err) + return fmt.Errorf("failed to read config: %v", err) } nodes := &corev1.NodeList{ @@ -303,7 +340,7 @@ clusters: Name: "cluster-1-node-2", }, Spec: corev1.NodeSpec{ - ProviderID: "proxmox://cluster-1/101", + ProviderID: providerID, }, }, }, @@ -311,14 +348,48 @@ clusters: kclient := fake.NewSimpleClientset(nodes) + cluster, err := proxmox.NewCluster(&cfg, &http.Client{}) + if err != nil { + return fmt.Errorf("failed to create proxmox cluster client: %v", err) + } + ts.s = &csi.ControllerService{ - Cluster: cluster, - Kclient: kclient, + Cluster: cluster, + Kclient: kclient, + Provider: cfg.Features.Provider, + } + + return nil +} + +// TestSuiteCSI runs all test configurations +func TestSuiteCSI(t *testing.T) { + configs := getTestConfigs() + for _, cfg := range configs { + // Create a new test suite for each configuration + ts := &baseCSITestSuite{} + + // Run the suite with the current configuration + suite.Run(t, &configuredTestSuite{ + baseCSITestSuite: ts, + configCase: cfg, + }) } } -func TestSuiteCCM(t *testing.T) { - suite.Run(t, new(csiTestSuite)) +// configuredTestSuite wraps the base suite with a specific configuration +type configuredTestSuite struct { + *baseCSITestSuite + configCase configTestCase +} + +func (ts *configuredTestSuite) SetupTest() { + setupMockResponders() + + err := ts.setupTestSuite(ts.configCase.config, ts.configCase.providerID) + if err != nil { + ts.T().Fatalf("Failed to setup test suite: %v", err) + } } func TestNewControllerService(t *testing.T) { @@ -333,7 +404,7 @@ func TestNewControllerService(t *testing.T) { } //nolint:dupl -func (ts *csiTestSuite) TestCreateVolume() { +func (ts *configuredTestSuite) TestCreateVolume() { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -682,7 +753,7 @@ func (ts *csiTestSuite) TestCreateVolume() { } //nolint:dupl -func (ts *csiTestSuite) TestDeleteVolume() { +func (ts *configuredTestSuite) TestDeleteVolume() { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -758,7 +829,7 @@ func (ts *csiTestSuite) TestDeleteVolume() { } } -func (ts *csiTestSuite) TestControllerServiceControllerGetCapabilities() { +func (ts *configuredTestSuite) TestControllerServiceControllerGetCapabilities() { resp, err := ts.s.ControllerGetCapabilities(context.Background(), &proto.ControllerGetCapabilitiesRequest{}) ts.Require().NoError(err) ts.Require().NotNil(resp) @@ -769,7 +840,7 @@ func (ts *csiTestSuite) TestControllerServiceControllerGetCapabilities() { } //nolint:dupl -func (ts *csiTestSuite) TestControllerPublishVolumeError() { +func (ts *configuredTestSuite) TestControllerPublishVolumeError() { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -903,7 +974,7 @@ func (ts *csiTestSuite) TestControllerPublishVolumeError() { } //nolint:dupl -func (ts *csiTestSuite) TestControllerUnpublishVolumeError() { +func (ts *configuredTestSuite) TestControllerUnpublishVolumeError() { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -975,19 +1046,19 @@ func (ts *csiTestSuite) TestControllerUnpublishVolumeError() { } } -func (ts *csiTestSuite) TestValidateVolumeCapabilities() { +func (ts *configuredTestSuite) TestValidateVolumeCapabilities() { _, err := ts.s.ValidateVolumeCapabilities(context.Background(), &proto.ValidateVolumeCapabilitiesRequest{}) ts.Require().Error(err) ts.Require().Equal(status.Error(codes.Unimplemented, ""), err) } -func (ts *csiTestSuite) TestListVolumes() { +func (ts *configuredTestSuite) TestListVolumes() { _, err := ts.s.ListVolumes(context.Background(), &proto.ListVolumesRequest{}) ts.Require().Error(err) ts.Require().Equal(status.Error(codes.Unimplemented, ""), err) } -func (ts *csiTestSuite) TestGetCapacity() { +func (ts *configuredTestSuite) TestGetCapacity() { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -1115,25 +1186,25 @@ func (ts *csiTestSuite) TestGetCapacity() { } } -func (ts *csiTestSuite) TestCreateSnapshot() { +func (ts *configuredTestSuite) TestCreateSnapshot() { _, err := ts.s.CreateSnapshot(context.Background(), &proto.CreateSnapshotRequest{}) ts.Require().Error(err) ts.Require().Equal(status.Error(codes.Unimplemented, ""), err) } -func (ts *csiTestSuite) TestDeleteSnapshot() { +func (ts *configuredTestSuite) TestDeleteSnapshot() { _, err := ts.s.DeleteSnapshot(context.Background(), &proto.DeleteSnapshotRequest{}) ts.Require().Error(err) ts.Require().Equal(status.Error(codes.Unimplemented, ""), err) } -func (ts *csiTestSuite) TestListSnapshots() { +func (ts *configuredTestSuite) TestListSnapshots() { _, err := ts.s.ListSnapshots(context.Background(), &proto.ListSnapshotsRequest{}) ts.Require().Error(err) ts.Require().Equal(status.Error(codes.Unimplemented, ""), err) } -func (ts *csiTestSuite) TestControllerExpandVolumeError() { +func (ts *configuredTestSuite) TestControllerExpandVolumeError() { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -1249,7 +1320,7 @@ func (ts *csiTestSuite) TestControllerExpandVolumeError() { } } -func (ts *csiTestSuite) TestControllerGetVolume() { +func (ts *configuredTestSuite) TestControllerGetVolume() { _, err := ts.s.ControllerGetVolume(context.Background(), &proto.ControllerGetVolumeRequest{}) ts.Require().Error(err) ts.Require().Equal(status.Error(codes.Unimplemented, ""), err) diff --git a/pkg/tools/nodes.go b/pkg/tools/nodes.go index 40250e6..1fb9250 100644 --- a/pkg/tools/nodes.go +++ b/pkg/tools/nodes.go @@ -18,9 +18,15 @@ package tools import ( "context" + "encoding/base64" "fmt" + "net/url" + "strings" - provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" + pxapi "github.com/Telmate/proxmox-api-go/proxmox" + + proxmox "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" + "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -89,13 +95,90 @@ func UncondonNodes(ctx context.Context, kclient *clientkubernetes.Clientset, nod } // ProxmoxVMID returns the Proxmox VM ID from the specified kubernetes node name. -func ProxmoxVMID(ctx context.Context, kclient clientkubernetes.Interface, nodeName string) (int, string, error) { +func ProxmoxVMID(ctx context.Context, kclient clientkubernetes.Interface, px *pxapi.Client, nodeName string, prov proxmox.Provider) (int, string, error) { node, err := kclient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) if err != nil { return 0, "", fmt.Errorf("failed to get node: %v", err) } + if prov == proxmox.ProviderCapmox { + return ProxmoxVMIDByProviderID(px, node.Spec.ProviderID) + } + vmID, err := provider.GetVMID(node.Spec.ProviderID) return vmID, node.Labels[corev1.LabelTopologyZone], err } + +// ProxmoxVMIDByProviderID find a VM by uuid in all Proxmox clusters. +func ProxmoxVMIDByProviderID(px *pxapi.Client, providerID string) (int, string, error) { + uuid := strings.TrimPrefix(providerID, "proxmox://") + + vm, err := findVMByUUID(px, uuid) + if err != nil { + return 0, "", err + } + + return vm.VmId(), vm.Node(), nil +} + +func findVMByUUID(px *pxapi.Client, uuid string) (*pxapi.VmRef, error) { + vms, err := px.GetResourceList("vm") + if err != nil { + return nil, fmt.Errorf("error get resources %v", err) + } + + for vmii := range vms { + vm, ok := vms[vmii].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("failed to cast response to map, vm: %v", vm) + } + + if vm["type"].(string) != "qemu" { //nolint:errcheck + continue + } + + vmr := pxapi.NewVmRef(int(vm["vmid"].(float64))) //nolint:errcheck + vmr.SetNode(vm["node"].(string)) //nolint:errcheck + vmr.SetVmType("qemu") + + config, err := px.GetVmConfig(vmr) + if err != nil { + return nil, err + } + + if config["smbios1"] != nil { + if getUUID(config["smbios1"].(string)) == uuid { //nolint:errcheck + return vmr, nil + } + } + } + + return nil, fmt.Errorf("vm with uuid '%s' not found", uuid) +} + +func getUUID(smbios string) string { + for _, l := range strings.Split(smbios, ",") { + if l == "" || l == "base64=1" { + continue + } + + parsedParameter, err := url.ParseQuery(l) + if err != nil { + return "" + } + + for k, v := range parsedParameter { + if k == "uuid" { + decodedString, err := base64.StdEncoding.DecodeString(v[0]) + if err != nil { + decodedString = []byte(v[0]) + } + + return string(decodedString) + } + } + } + + return "" +}