From c465ccb57d98a8d98695251885527ea9a2fc7b2b Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Mon, 8 Feb 2021 14:43:11 -0500 Subject: [PATCH] This patch includes vlan trunking for Intel and Mellanox: (1) https://github.com/k8snetworkplumbingwg/sriov-cni/pull/149, plus (2) extension for Mellanox Signed-off-by: Jing Zhang --- docs/configuration-reference.md | 3 +- go.mod | 1 + go.sum | 2 + pkg/config/config.go | 6 + pkg/factory/factory.go | 34 +++++ pkg/factory/factory_suite_test.go | 32 +++++ pkg/factory/factory_test.go | 26 ++++ pkg/providers/intel.go | 173 ++++++++++++++++++++++++++ pkg/providers/mellanox.go | 123 ++++++++++++++++++ pkg/providers/providers_suite_test.go | 32 +++++ pkg/providers/providers_test.go | 118 ++++++++++++++++++ pkg/sriov/sriov.go | 41 +++++- pkg/types/types.go | 21 ++++ pkg/utils/testing.go | 16 +++ pkg/utils/utils.go | 131 +++++++++++++++++++ pkg/utils/utils_test.go | 108 ++++++++++++++++ 16 files changed, 865 insertions(+), 2 deletions(-) create mode 100644 pkg/factory/factory.go create mode 100644 pkg/factory/factory_suite_test.go create mode 100644 pkg/factory/factory_test.go create mode 100644 pkg/providers/intel.go create mode 100644 pkg/providers/mellanox.go create mode 100644 pkg/providers/providers_suite_test.go create mode 100644 pkg/providers/providers_test.go diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index dd7c28f18..079cf9e6f 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -41,7 +41,8 @@ An SR-IOV CNI config with each field filled out looks like: "trust": "on", "link_state": "enable", "logLevel": "debug", - "logFile": "/tmp/sriov.log" + "logFile": "/tmp/sriov.log", + "vlan_trunk": "10,12,20-30,1100" } ``` diff --git a/go.mod b/go.mod index 4e4bab39f..1f0e7373a 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/k8snetworkplumbingwg/sriov-cni go 1.22.4 require ( + github.com/Masterminds/semver v1.5.0 github.com/containernetworking/cni v1.2.0-rc0.0.20240317203738-a448e71e9867 github.com/containernetworking/plugins v1.4.2-0.20240312120516-c860b78de419 github.com/k8snetworkplumbingwg/cni-log v0.0.0-20230801160229-b6e062c9e0f2 diff --git a/go.sum b/go.sum index ce8a5b328..9c92570d2 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/containernetworking/cni v1.2.0-rc0.0.20240317203738-a448e71e9867 h1:DQ9iOvlXFOn+sJfbdvXyGISf/4xHNFGxJltq4mixK00= github.com/containernetworking/cni v1.2.0-rc0.0.20240317203738-a448e71e9867/go.mod h1:Lt0TQcZQVDju64fYxUhDziTgXCDe3Olzi9I4zZJLWHg= github.com/containernetworking/plugins v1.4.2-0.20240312120516-c860b78de419 h1:mvCb6RL9/tZwgXnkYNQQk6JDtLgHdtFde8uVm7VKg04= diff --git a/pkg/config/config.go b/pkg/config/config.go index e15fa3829..bcfe385d0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -137,6 +137,12 @@ func LoadConf(bytes []byte) (*sriovtypes.NetConf, error) { return nil, fmt.Errorf("LoadConf(): invalid link_state value: %s", n.LinkState) } + if n.VlanTrunk != "" { + if err := utils.ValidateVlanTrunkValue(n.VlanTrunk); err != nil { + return nil, fmt.Errorf("LoadConf(): invalid vlan_trunk value: %s", n.VlanTrunk) + } + } + return n, nil } diff --git a/pkg/factory/factory.go b/pkg/factory/factory.go new file mode 100644 index 000000000..05d354c16 --- /dev/null +++ b/pkg/factory/factory.go @@ -0,0 +1,34 @@ +package factory + +import ( + "fmt" + + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/providers" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/types" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils" +) + +const ( + //IntelProviderID Intel vendor ID + IntelProviderID = "0x8086" + //MellanoxProviderID Mellanox vendor ID + MellanoxProviderID = "0x15b3" +) + +// GetProviderConfig get Config for specific NIC +func GetProviderConfig(deviceID string) (types.VlanTrunkProviderConfig, error) { + vendor, err := utils.GetVendorID(deviceID) + if err != nil { + return nil, fmt.Errorf("GetVendorID Error: %q", err) + } + + switch vendor { + case IntelProviderID: + return providers.NewIntelTrunkProviderConfig(), nil + case MellanoxProviderID: + return providers.NewMellanoxTrunkProviderConfig(), nil + default: + return nil, fmt.Errorf("Not supported vendor: %q", vendor) + } + +} diff --git a/pkg/factory/factory_suite_test.go b/pkg/factory/factory_suite_test.go new file mode 100644 index 000000000..e561954f0 --- /dev/null +++ b/pkg/factory/factory_suite_test.go @@ -0,0 +1,32 @@ +package factory + +import ( + "testing" + + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func check(e error) { + if e != nil { + panic(e) + } +} +func TestFactory(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Factory Suite") +} + +var _ = BeforeSuite(func() { + // create test sys tree + err := utils.CreateTmpSysFs() + check(err) +}) + +var _ = AfterSuite(func() { + var err error + err = utils.RemoveTmpSysFs() + check(err) +}) diff --git a/pkg/factory/factory_test.go b/pkg/factory/factory_test.go new file mode 100644 index 000000000..bcfcc5823 --- /dev/null +++ b/pkg/factory/factory_test.go @@ -0,0 +1,26 @@ +package factory + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Factory", func() { + + Context("Checking GetProviderConfig function", func() { + It("Assuming existing vf", func() { + _, err := GetProviderConfig("0000:af:06.1") + // Expect(result).To(Equal(&providers.IntelTrunkProviderConfig{ProviderName: "Intel"})) + Expect(err).NotTo(HaveOccurred(), "Existing vf should not return an error") + }) + It("Assuming existing vf", func() { + _, err := GetProviderConfig("0000:cf:06.0") + // Expect(result).To(Equal(&providers.MellanoxTrunkProviderConfig{ProviderName: "Mellanox"})) + Expect(err).NotTo(HaveOccurred(), "Existing vf should not return an error, unless the vendor is not supported") + }) + It("Assuming not existing vf", func() { + _, err := GetProviderConfig("0000:af:07.0") + Expect(err).To(HaveOccurred(), "Not existing vf should return an error") + }) + }) +}) diff --git a/pkg/providers/intel.go b/pkg/providers/intel.go new file mode 100644 index 000000000..dd3f7a8fc --- /dev/null +++ b/pkg/providers/intel.go @@ -0,0 +1,173 @@ +package providers + +import ( + "bytes" + "fmt" + "io/ioutil" + "os/exec" + "strconv" + "strings" + + "github.com/Masterminds/semver" + + sriovtypes "github.com/k8snetworkplumbingwg/sriov-cni/pkg/types" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils" +) + +var execCommand = exec.Command + +//IntelTrunkProviderConfig stores name of the provider +type IntelTrunkProviderConfig struct { + ProviderName string + VlanData string +} + +//NewIntelTrunkProviderConfig creates new Intel provider configuraton +func NewIntelTrunkProviderConfig() sriovtypes.VlanTrunkProviderConfig { + return &IntelTrunkProviderConfig{ + ProviderName: "Intel", + } +} + +//InitConfig initializes provider configuration for given trunking ranges +func (p *IntelTrunkProviderConfig) InitConfig(vlanRanges *sriovtypes.VlanTrunkRangeData) { + p.GetVlanData(vlanRanges) + return +} + +//ApplyConfig applies provider configuration +func (p *IntelTrunkProviderConfig) ApplyConfig(conf *sriovtypes.NetConf) error { + if trunkingSupported := CheckTrunkSupport(); trunkingSupported == false { + return fmt.Errorf("Vlan trunking supported only by i40e version 2.7.11 and higher, please upgrade your driver") + } + + if err := AddVlanFiltering(p.VlanData, conf.Master, conf.VFID); err != nil { + return err + } + + return nil +} + +//RemoveConfig removes configuration +func (p *IntelTrunkProviderConfig) RemoveConfig(conf *sriovtypes.NetConf) error { + if err := RemoveVlanFiltering(p.VlanData, conf.Master, conf.VFID); err != nil { + return err + } + + return nil +} + +//GetVlanData converts vlanRanges.VlanTrunkRanges into string +func (p *IntelTrunkProviderConfig) GetVlanData(vlanRanges *sriovtypes.VlanTrunkRangeData) { + vlanData := "" + var start, end string + + for i, vlanRange := range vlanRanges.VlanTrunkRanges { + + start = strconv.Itoa(int(vlanRange.Start)) + end = strconv.Itoa(int(vlanRange.End)) + vlanData = vlanData + start + + if start != end { + vlanData = vlanData + "-" + end + } + if i < len(vlanRanges.VlanTrunkRanges)-1 { + vlanData = vlanData + "," + } + + } + p.VlanData = vlanData + return +} + +//AddVlanFiltering writes "add [trunking ranges]" to trunk file +func AddVlanFiltering(vlanData, pfName string, vfid int) error { + addTrunk := "add " + vlanData + trunkFile := fmt.Sprintf(utils.TrunkFileDirectory, pfName, vfid) + + errwrite := ioutil.WriteFile(trunkFile, []byte(addTrunk), 0644) + if errwrite != nil { + return fmt.Errorf("f.Write: %q", errwrite) + } + + infraInterfaces, infraVlans, err := utils.GetInfraVlanData() + if err == nil { + for _, pf := range infraInterfaces { + if pf == pfName { + vlanData := "" + for _, vlan := range infraVlans { + if vlanData == "" { + vlanData = fmt.Sprintf("%d", vlan) + } else { + vlanData = fmt.Sprintf("%s,%d", vlanData, vlan) + } + } + if vlanData != "" { + removeTrunk := "rem " + vlanData + errwrite := ioutil.WriteFile(trunkFile, []byte(removeTrunk), 0644) + if errwrite != nil { + return fmt.Errorf("f.Write: %q", errwrite) + } + } + } + } + } + + return nil +} + +//RemoveVlanFiltering writes "rem [trunking ranges]" to trunk file +func RemoveVlanFiltering(vlanData, pfName string, vfid int) error { + removeTrunk := "rem " + vlanData + trunkFile := fmt.Sprintf(utils.TrunkFileDirectory, pfName, vfid) + + errwrite := ioutil.WriteFile(trunkFile, []byte(removeTrunk), 0644) + if errwrite != nil { + return fmt.Errorf("f.Write: %q", errwrite) + } + + return nil +} + +// CheckTrunkSupport checks installed driver version; trunking is supported for version 2.7.11 and higher +func CheckTrunkSupport() bool { + var stdout bytes.Buffer + modinfoCmd := "modinfo -F version i40e" + cmd := execCommand("sh", "-c", modinfoCmd) + cmd.Stdout = &stdout + + if err := cmd.Run(); err != nil { + fmt.Printf("modinfo returned error: %v %s", err, stdout.String()) + return false + } + + stdoutSplit := strings.Split(stdout.String(), "\n") + if len(stdoutSplit) == 0 { + fmt.Printf("unexpected output after parsing driver version: %s", stdout.String()) + return false + } + driverVersion := stdoutSplit[0] + numDots := strings.Count(driverVersion, ".") + if numDots < 2 { + fmt.Printf("unexpected driver version: %s", driverVersion) + return false + } + //truncate driver version to only major.minor.patch version format length to ensure semver compatibility + if numDots > 2 { + truncVersion := strings.Split(driverVersion, ".")[:3] + driverVersion = strings.Join(truncVersion, ".") + } + + v1, _ := semver.NewVersion("2.7.11") + v2, err := semver.NewVersion(driverVersion) + if err != nil { + fmt.Printf("invalid version error: %v %s", err, driverVersion) + return false + } + + if v2.Compare(v1) >= 0 { + return true + } + + return false +} diff --git a/pkg/providers/mellanox.go b/pkg/providers/mellanox.go new file mode 100644 index 000000000..8eaa38d7e --- /dev/null +++ b/pkg/providers/mellanox.go @@ -0,0 +1,123 @@ +package providers + +import ( + "bytes" + "fmt" + sriovtypes "github.com/k8snetworkplumbingwg/sriov-cni/pkg/types" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils" + "io/ioutil" + "strconv" +) + +//MellanoxTrunkProviderConfig stores name of the provider +type MellanoxTrunkProviderConfig struct { + ProviderName string + VlanData []string +} + +//NewMellanoxTrunkProviderConfig creates new Mellanox provider configuraton +func NewMellanoxTrunkProviderConfig() sriovtypes.VlanTrunkProviderConfig { + return &MellanoxTrunkProviderConfig{ + ProviderName: "Mellanox", + } +} + +//InitConfig initializes provider configuration for given trunking ranges +func (p *MellanoxTrunkProviderConfig) InitConfig(vlanRanges *sriovtypes.VlanTrunkRangeData) { + p.GetVlanData(vlanRanges) + return +} + +//ApplyConfig applies provider configuration +func (p *MellanoxTrunkProviderConfig) ApplyConfig(conf *sriovtypes.NetConf) error { + if trunkingSupported := CheckVgtPlusSupport(); trunkingSupported == false { + return fmt.Errorf("Vlan trunking is only supported by mlx5_core") + } + + if err := EnableVgtPlus(p.VlanData, conf.Master, conf.VFID); err != nil { + return err + } + + return nil +} + +//RemoveConfig removes configuration +func (p *MellanoxTrunkProviderConfig) RemoveConfig(conf *sriovtypes.NetConf) error { + if err := DisableVgtPlus(p.VlanData, conf.Master, conf.VFID); err != nil { + return err + } + + return nil +} + +//GetVlanData converts vlanRanges.VlanTrunkRanges into string +func (p *MellanoxTrunkProviderConfig) GetVlanData(vlanRanges *sriovtypes.VlanTrunkRangeData) { + var vlanData []string + var start, end string + + for _, vlanRange := range vlanRanges.VlanTrunkRanges { + start = strconv.Itoa(int(vlanRange.Start)) + end = strconv.Itoa(int(vlanRange.End)) + vlanData = append(vlanData, start+" "+end) + } + p.VlanData = vlanData + return +} + +//EnableVgtPlus writes "add " to trunk file +func EnableVgtPlus(vlanData []string, pfName string, vfid int) error { + trunkFile := fmt.Sprintf(utils.TrunkFileDirectory, pfName, vfid) + for _, vlans := range vlanData { + addTrunk := "add " + vlans + errwrite := ioutil.WriteFile(trunkFile, []byte(addTrunk), 0644) + if errwrite != nil { + return fmt.Errorf("f.Write: %q", errwrite) + } + } + + infraInterfaces, infraVlans, err := utils.GetInfraVlanData() + if err == nil { + for _, pf := range infraInterfaces { + if pf == pfName { + for _, vlan := range infraVlans { + removeTrunk := fmt.Sprintf("rem %d %d", vlan, vlan) + errwrite := ioutil.WriteFile(trunkFile, []byte(removeTrunk), 0644) + if errwrite != nil { + return fmt.Errorf("f.Write: %q", errwrite) + } + } + } + } + } + + return nil +} + +//DisableVgtPlus writes "rem " to trunk file +func DisableVgtPlus(vlanData []string, pfName string, vfid int) error { + trunkFile := fmt.Sprintf(utils.TrunkFileDirectory, pfName, vfid) + for _, vlans := range vlanData { + removeTrunk := "rem " + vlans + errwrite := ioutil.WriteFile(trunkFile, []byte(removeTrunk), 0644) + if errwrite != nil { + return fmt.Errorf("f.Write: %q", errwrite) + } + } + + return nil +} + +// CheckVgtPlusSupport checks mlx5_core is installed +func CheckVgtPlusSupport() bool { + var stdout bytes.Buffer + modinfoCmd := "modinfo -F version mlx5_core" + cmd := execCommand("sh", "-c", modinfoCmd) + cmd.Stdout = &stdout + + if err := cmd.Run(); err != nil { + fmt.Printf("modinfo returned error: %v %s", err, stdout.String()) + return false + } + + return true +} diff --git a/pkg/providers/providers_suite_test.go b/pkg/providers/providers_suite_test.go new file mode 100644 index 000000000..27efe023a --- /dev/null +++ b/pkg/providers/providers_suite_test.go @@ -0,0 +1,32 @@ +package providers + +import ( + "testing" + + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func check(e error) { + if e != nil { + panic(e) + } +} +func TestUtils(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Providers Suite") +} + +var _ = BeforeSuite(func() { + // create test sys tree + err := utils.CreateTmpSysFs() + check(err) +}) + +var _ = AfterSuite(func() { + var err error + err = utils.RemoveTmpSysFs() + check(err) +}) diff --git a/pkg/providers/providers_test.go b/pkg/providers/providers_test.go new file mode 100644 index 000000000..77ad74b2b --- /dev/null +++ b/pkg/providers/providers_test.go @@ -0,0 +1,118 @@ +package providers + +import ( + "os/exec" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + sriovtypes "github.com/k8snetworkplumbingwg/sriov-cni/pkg/types" +) + +func FakeNoTrunkSupport(command string, args ...string) *exec.Cmd { + command = "echo" + args[0] = "1.2.3" + cmd := exec.Command(command, args[0]) + + return cmd +} + +func FakeTrunkSupport(command string, args ...string) *exec.Cmd { + command = "echo" + args[0] = "2.7.11" + cmd := exec.Command(command, args[0]) + + return cmd +} + +var _ = Describe("Providers", func() { + Context("Checking Init/Apply/RemoveConfig function", func() { + It("Assuming valid config", func() { + execCommand = FakeTrunkSupport + p := NewIntelTrunkProviderConfig() + vlanRanges := &sriovtypes.VlanTrunkRangeData{ + VlanTrunkRanges: []sriovtypes.VlanTrunkRange{{Start: 1, End: 3}, {Start: 5, End: 10}, {Start: 13, End: 13}}, + } + netConf := &sriovtypes.NetConf{} + netConf.Master = "enp175s6" + netConf.VFID = 0 + p.InitConfig(vlanRanges) + + err := p.ApplyConfig(netConf) + Expect(err).NotTo(HaveOccurred()) + + err = p.RemoveConfig(netConf) + Expect(err).NotTo(HaveOccurred()) + }) + }) + Context("Checking GetVlanData function", func() { + It("Assuming correct VlanTrunkRangeData", func() { + vlanRanges := &sriovtypes.VlanTrunkRangeData{ + VlanTrunkRanges: []sriovtypes.VlanTrunkRange{{Start: 1, End: 3}, {Start: 5, End: 10}, {Start: 13, End: 13}}, + } + p := &IntelTrunkProviderConfig{ + ProviderName: "Intel", + } + p.GetVlanData(vlanRanges) + Expect(p.VlanData).To(Equal("1-3,5-10,13")) + }) + }) + Context("Checking AddVlanFiltering function", func() { + It("Assuming existing vf", func() { + vlanData := "2,6,100-200" + pfname := "enp175s6" + vfid := 0 + err := AddVlanFiltering(vlanData, pfname, vfid) + Expect(err).NotTo(HaveOccurred()) + }) + It("Assuming non-existing vf", func() { + vlanData := "2,6,100-200" + pfname := "enp175s16" + vfid := 0 + err := AddVlanFiltering(vlanData, pfname, vfid) + Expect(err).To(HaveOccurred()) + }) + It("Assuming null byte injection", func() { + vlanData := "2,6,100-200" + pfname := "enp175s1%006" + vfid := 0 + err := AddVlanFiltering(vlanData, pfname, vfid) + Expect(err).To(HaveOccurred()) + }) + }) + Context("Checking RemoveVlanFiltering function", func() { + It("Assuming existing vf", func() { + vlanData := "2,6,100-200" + pfname := "enp175s0f1" + vfid := 1 + err := RemoveVlanFiltering(vlanData, pfname, vfid) + Expect(err).NotTo(HaveOccurred()) + }) + It("Assuming non-existing vf", func() { + vlanData := "2,6,100-200" + pfname := "enp175s16" + vfid := 0 + err := RemoveVlanFiltering(vlanData, pfname, vfid) + Expect(err).To(HaveOccurred()) + }) + It("Assuming null byte injection", func() { + vlanData := "2,6,100-200" + pfname := "enp175s1%006" + vfid := 0 + err := RemoveVlanFiltering(vlanData, pfname, vfid) + Expect(err).To(HaveOccurred()) + }) + }) + Context("Checking CheckTrunkSupport function", func() { + It("Assuming version higher or equal to 2.7.11", func() { + execCommand = FakeTrunkSupport + result := CheckTrunkSupport() + Expect(result).To(Equal(true)) + }) + It("Assuming version lower than 2.7.11", func() { + execCommand = FakeNoTrunkSupport + result := CheckTrunkSupport() + Expect(result).To(Equal(false)) + }) + }) +}) diff --git a/pkg/sriov/sriov.go b/pkg/sriov/sriov.go index f5c57970f..4fe38e60c 100644 --- a/pkg/sriov/sriov.go +++ b/pkg/sriov/sriov.go @@ -5,6 +5,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/factory" "github.com/k8snetworkplumbingwg/sriov-cni/pkg/logging" sriovtypes "github.com/k8snetworkplumbingwg/sriov-cni/pkg/types" "github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils" @@ -335,6 +336,26 @@ func (s *sriovManager) ApplyVFConfig(conf *sriovtypes.NetConf) error { pfMtu := pfLink.Attrs().MTU conf.MTU = &pfMtu + // 7. Set vlan trunking + if conf.VlanTrunk != "" { + vlanTrunkRange, err := utils.GetVlanTrunkRange(conf.VlanTrunk) + if err != nil { + return fmt.Errorf("GetVlanTrunkRange Error: %q", err) + } + + vlanTrunkProviderConfig, err := factory.GetProviderConfig(conf.DeviceID) + if err != nil { + return fmt.Errorf("GetProviderConfig Error: %q", err) + } + + vlanTrunkProviderConfig.InitConfig(&vlanTrunkRange) + + if err := vlanTrunkProviderConfig.ApplyConfig(conf); err != nil { + return fmt.Errorf("ApplyConfig Error: %q", err) + } + + } + return nil } @@ -366,7 +387,7 @@ func (s *sriovManager) ResetVFConfig(conf *sriovtypes.NetConf) error { conf.OrigVfState.VlanProto = sriovtypes.VlanProtoInt[sriovtypes.Proto8021q] } - if conf.Vlan != nil { + if conf.Vlan != nil && conf.VlanTrunk == "" { if err = s.nLink.LinkSetVfVlanQosProto(pfLink, conf.VFID, conf.OrigVfState.Vlan, conf.OrigVfState.VlanQoS, conf.OrigVfState.VlanProto); err != nil { return fmt.Errorf("failed to set vf %d vlan configuration - id %d, qos %d and proto %d: %v", conf.VFID, conf.OrigVfState.Vlan, conf.OrigVfState.VlanQoS, conf.OrigVfState.VlanProto, err) } @@ -410,5 +431,23 @@ func (s *sriovManager) ResetVFConfig(conf *sriovtypes.NetConf) error { } } + // Restore VLAN Trunk + if conf.VlanTrunk != "" { + vlanTrunkRange, err := utils.GetVlanTrunkRange(conf.VlanTrunk) + if err != nil { + return fmt.Errorf("GetVlanTrunkRange Error: %q", err) + } + + vlanTrunkProviderConfig, err := factory.GetProviderConfig(conf.DeviceID) + if err != nil { + return fmt.Errorf("GetProviderConfig Error: %q", err) + } + + vlanTrunkProviderConfig.GetVlanData(&vlanTrunkRange) + if err := vlanTrunkProviderConfig.RemoveConfig(conf); err != nil { + return fmt.Errorf("RemoveConfig Error: %q", err) + } + } + return nil } diff --git a/pkg/types/types.go b/pkg/types/types.go index 07b30f83f..9fcabc54f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -3,6 +3,7 @@ package types import ( "encoding/json" "fmt" + "github.com/containernetworking/cni/pkg/types" "github.com/vishvananda/netlink" ) @@ -64,6 +65,7 @@ type SriovNetConf struct { SpoofChk string `json:"spoofchk,omitempty"` // on|off Trust string `json:"trust,omitempty"` // on|off LinkState string `json:"link_state,omitempty"` // auto|enable|disable + VlanTrunk string `json:"vlan_trunk,omitempty"` // vlan trunking RuntimeConfig struct { Mac string `json:"mac,omitempty"` } `json:"runtimeConfig,omitempty"` @@ -103,3 +105,22 @@ func (n *NetConf) MarshalJSON() ([]byte, error) { return sriovNetConfBytes, nil } + +// VlanTrunkProviderConfig provdes methods for provider configuration +type VlanTrunkProviderConfig interface { + InitConfig(vlanRanges *VlanTrunkRangeData) + ApplyConfig(conf *NetConf) error + RemoveConfig(conf *NetConf) error + GetVlanData(vlanRanges *VlanTrunkRangeData) +} + +// VlanTrunkRange strores trunking range +type VlanTrunkRange struct { + Start uint + End uint +} + +// VlanTrunkRangeData stores an array of VlanTrunkRange +type VlanTrunkRangeData struct { + VlanTrunkRanges []VlanTrunkRange +} diff --git a/pkg/utils/testing.go b/pkg/utils/testing.go index 7db0c19de..adb851cc5 100644 --- a/pkg/utils/testing.go +++ b/pkg/utils/testing.go @@ -31,17 +31,25 @@ var ts = tmpSysFs{ "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/net/enp175s0f1", "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.0/net/enp175s6", "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1/net/enp175s7", + "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:00.1/net/enp185s0f1", + "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:06.0/net/enp185s6", "sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0/net/ens1", "sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0/net/ens1d1", }, fileList: map[string][]byte{ "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/sriov_numvfs": []byte("2"), + "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:00.1/sriov_numvfs": []byte("1"), "sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0/sriov_numvfs": []byte("0"), + "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1/vendor": []byte("0x8086"), + "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:06.0/vendor": []byte("0x15b3"), + "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/trunk": []byte("1000"), }, netSymlinks: map[string]string{ "sys/class/net/enp175s0f1": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/net/enp175s0f1", "sys/class/net/enp175s6": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.0/net/enp175s6", "sys/class/net/enp175s7": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1/net/enp175s7", + "sys/class/net/enp185s0f1": "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:00.1/net/enp185s0f1", + "sys/class/net/enp185s6": "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:06.0/net/enp185s6", "sys/class/net/ens1": "sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0/net/ens1", "sys/class/net/ens1d1": "sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0/net/ens1d1", }, @@ -49,12 +57,16 @@ var ts = tmpSysFs{ "sys/class/net/enp175s0f1/device": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1", "sys/class/net/enp175s6/device": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.0", "sys/class/net/enp175s7/device": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1", + "sys/class/net/enp185s0f1/device": "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:00.1", + "sys/class/net/enp185s6/device": "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:06.0", "sys/class/net/ens1/device": "sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0", "sys/class/net/ens1d1/device": "sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0", "sys/bus/pci/devices/0000:af:00.1": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1", "sys/bus/pci/devices/0000:af:06.0": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.0", "sys/bus/pci/devices/0000:af:06.1": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1", + "sys/bus/pci/devices/0000:cf:00.1": "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:00.1", + "sys/bus/pci/devices/0000:cf:06.0": "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:06.0", "sys/bus/pci/devices/0000:05:00.0": "sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0", }, vfSymlinks: map[string]string{ @@ -63,6 +75,9 @@ var ts = tmpSysFs{ "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/virtfn1": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1", "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1/physfn": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1", + + "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:00.1/virtfn1": "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:06.0", + "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:06.0/physfn": "sys/devices/pci0000:ae/0000:ae:00.0/0000:cf:00.1", }, } @@ -111,6 +126,7 @@ func CreateTmpSysFs() error { SysBusPci = filepath.Join(ts.dirRoot, SysBusPci) NetDirectory = filepath.Join(ts.dirRoot, NetDirectory) + TrunkFileDirectory = filepath.Join(ts.dirRoot, "sys/class/net/%s/device/trunk") return nil } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 3a64a4222..c7096854c 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -4,9 +4,13 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/types" + "io/ioutil" "net" "os" + "os/exec" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -18,6 +22,8 @@ var ( sriovConfigured = "/sriov_numvfs" // NetDirectory sysfs net directory NetDirectory = "/sys/class/net" + // DevSubDirectory device subdirectory + DevSubDirectory = "/device/driver" // SysBusPci is sysfs pci device directory SysBusPci = "/sys/bus/pci/devices" // SysV4ArpNotify is the sysfs IPv4 ARP Notify directory @@ -26,6 +32,10 @@ var ( SysV6NdiscNotify = "/proc/sys/net/ipv6/conf/" // UserspaceDrivers is a list of driver names that don't have netlink representation for their devices UserspaceDrivers = []string{"vfio-pci", "uio_pci_generic", "igb_uio"} + //ExecCommand used for os.exec + execCommand = exec.Command + // TrunkFileDirectory trunk file directoy + TrunkFileDirectory = "/sys/class/net/%s/device/sriov/%d/trunk" ) // EnableArpAndNdiscNotify enables IPv4 arp_notify and IPv6 ndisc_notify for netdev @@ -414,3 +424,124 @@ func Retry(retries int, sleep time.Duration, f func() error) error { } return err } + +// ValidateVlanTrunkValue validates vlan trunking input +func ValidateVlanTrunkValue(vlanTrunk string) error { + validTrunkValue := regexp.MustCompile("^[0-9]+([,\\-][0-9]+)*$") + if !validTrunkValue.MatchString(vlanTrunk) { + return fmt.Errorf("Invalid vlan_trunk value") + } + return nil +} + +// GetVlanTrunkRange creates VlanTrunkRangeData from vlanTrunkString +func GetVlanTrunkRange(vlanTrunkString string) (types.VlanTrunkRangeData, error) { + var vlanRange = []types.VlanTrunkRange{} + trunkingRanges := strings.Split(vlanTrunkString, ",") + + for _, r := range trunkingRanges { + values := strings.Split(r, "-") + v1, errconv1 := strconv.Atoi(values[0]) + v2, errconv2 := strconv.Atoi(values[len(values)-1]) + + if errconv1 != nil || errconv2 != nil { + return types.VlanTrunkRangeData{}, fmt.Errorf("Trunk range error: invalid values") + } + + v := types.VlanTrunkRange{ + Start: uint(v1), + End: uint(v2), + } + + vlanRange = append(vlanRange, v) + } + if err := ValidateVlanTrunkRange(vlanRange); err != nil { + return types.VlanTrunkRangeData{}, err + } + + vlanRanges := types.VlanTrunkRangeData{ + VlanTrunkRanges: vlanRange, + } + return vlanRanges, nil + +} + +// ValidateVlanTrunkRange checks if given vlan trunking ranges are of correct form +func ValidateVlanTrunkRange(vlanRanges []types.VlanTrunkRange) error { + + for i, r1 := range vlanRanges { + if r1.Start > r1.End { + return fmt.Errorf("Invalid VlanTrunk range values") + } + + if r1.Start < 1 || r1.End > 4094 { + return fmt.Errorf("Invalid VlanTrunk range values") + } + + for j, r2 := range vlanRanges { + if r1.End > r2.Start && i < j { + return fmt.Errorf("Invalid VlanTrunk range values") + } + } + + } + return nil +} + +// GetVendorID returns ID of installed vendor +func GetVendorID(deviceID string) (string, error) { + path := filepath.Join(SysBusPci, deviceID, "vendor") + + readVendor, err := ioutil.ReadFile(path) + if err != nil { + return "", fmt.Errorf("Error reading vendor file %q, %q", path, err) + } + + vendorCode := strings.Split(string(readVendor), "\n")[0] + + return vendorCode, nil +} + +// GetInfraVlanData returns vlan ranges used by cloud infra-structure +func GetInfraVlanData() ([]string, []int, error) { + type Member struct { + Type string `json:"type"` + Name string `json:"name"` + } + type Network struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + Members []Member `json:"members,omitempty"` + VlanID int `json:"vlan_id,omitempty"` + Device string `json:"device,omitempty"` + } + type NetworkConfig struct { + Networks []Network `json:"network_config"` + } + + path := "/etc/os-net-config/config.json" + byteValue, err := ioutil.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("Error reading os-net-config json file %q, %q", path, err) + } + + var networkConfig NetworkConfig + json.Unmarshal(byteValue, &networkConfig) + + var infraInterfaces []string + var infraVlans []int + for _, network := range networkConfig.Networks { + if network.Name == "infra-bond" { + for _, member := range network.Members { + if member.Type == "interface" { + infraInterfaces = append(infraInterfaces, member.Name) + } + } + } + if network.Type == "vlan" && network.Device == "infra-bond" { + infraVlans = append(infraVlans, network.VlanID) + } + } + + return infraInterfaces, infraVlans, nil +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index bcd891aa0..fea052aab 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -233,4 +233,112 @@ var _ = Describe("Utils", func() { Expect(netconf.DNS.Domain).To(Equal(newNetConf.DNS.Domain)) }) }) + Context("Checking ValidateVlanTrunkValue function", func() { + It("Assuming valid vlan range", func() { + vlanTrunk := "1,4-6,10" + err := ValidateVlanTrunkValue(vlanTrunk) + Expect(err).NotTo(HaveOccurred(), "Valid vlan range should not return an error") + }) + It("Assuming invalid vlan range", func() { + vlanTrunk := "3,2-6%00,10" + err := ValidateVlanTrunkValue(vlanTrunk) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming invalid vlan range", func() { + vlanTrunk := "a,2-fed,10" + err := ValidateVlanTrunkValue(vlanTrunk) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming invalid vlan range", func() { + vlanTrunk := "Ab-c,d-g,XXX" + err := ValidateVlanTrunkValue(vlanTrunk) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming invalid vlan range", func() { + vlanTrunk := " " + err := ValidateVlanTrunkValue(vlanTrunk) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming invalid vlan range", func() { + vlanTrunk := "" + err := ValidateVlanTrunkValue(vlanTrunk) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming invalid vlan range", func() { + vlanTrunk := "0/23,5+10" + err := ValidateVlanTrunkValue(vlanTrunk) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + }) + Context("Checking GetVlanTrunkRange function", func() { + It("Assuming valid vlan range", func() { + result, err := GetVlanTrunkRange("1,4-6,10") + expectedRangeData := sriovtypes.VlanTrunkRangeData{ + VlanTrunkRanges: []sriovtypes.VlanTrunkRange{{Start: 1, End: 1}, {Start: 4, End: 6}, {Start: 10, End: 10}}, + } + Expect(result).To(Equal(expectedRangeData)) + Expect(err).NotTo(HaveOccurred(), "Valid vlan range should not return an error") + }) + It("Assuming not valid vlan range", func() { + _, err := GetVlanTrunkRange("3,2-6,10") + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming not valid vlan range", func() { + _, err := GetVlanTrunkRange("a,2-fed,10") + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + }) + Context("Checking ValidateVlanTrunkRange function", func() { + It("Assuming valid vlan range", func() { + testRangeData := sriovtypes.VlanTrunkRangeData{ + VlanTrunkRanges: []sriovtypes.VlanTrunkRange{{Start: 1, End: 1}, {Start: 4, End: 6}, {Start: 10, End: 10}}, + } + err := ValidateVlanTrunkRange(testRangeData.VlanTrunkRanges) + Expect(err).NotTo(HaveOccurred(), "Valid vlan range should not return an error") + }) + It("Assuming invalid vlan range", func() { + testRangeData := sriovtypes.VlanTrunkRangeData{ + VlanTrunkRanges: []sriovtypes.VlanTrunkRange{{Start: 3, End: 3}, {Start: 2, End: 6}, {Start: 10, End: 10}}, + } + err := ValidateVlanTrunkRange(testRangeData.VlanTrunkRanges) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming invalid vlan range", func() { + testRangeData := sriovtypes.VlanTrunkRangeData{ + VlanTrunkRanges: []sriovtypes.VlanTrunkRange{{Start: 5, End: 4}, {Start: 10, End: 10}}, + } + err := ValidateVlanTrunkRange(testRangeData.VlanTrunkRanges) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming invalid vlan range", func() { + testRangeData := sriovtypes.VlanTrunkRangeData{ + VlanTrunkRanges: []sriovtypes.VlanTrunkRange{{Start: 0, End: 0}, {Start: 3, End: 10}}, + } + err := ValidateVlanTrunkRange(testRangeData.VlanTrunkRanges) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + It("Assuming invalid vlan range", func() { + testRangeData := sriovtypes.VlanTrunkRangeData{ + VlanTrunkRanges: []sriovtypes.VlanTrunkRange{{Start: 1, End: 1}, {Start: 4091, End: 4095}}, + } + err := ValidateVlanTrunkRange(testRangeData.VlanTrunkRanges) + Expect(err).To(HaveOccurred(), "Invalid vlan range should return an error") + }) + }) + Context("Checking ReadVendorFile function", func() { + It("Assuming existing vf", func() { + result, err := GetVendorID("0000:af:06.1") + Expect(result).To(Equal("0x8086")) + Expect(err).NotTo(HaveOccurred()) + }) + It("Assuming existing vf", func() { + result, err := GetVendorID("0000:cf:06.0") + Expect(result).To(Equal("0x15b3")) + Expect(err).NotTo(HaveOccurred()) + }) + It("Assuming not existing vf", func() { + _, err := GetVendorID("0000:af:07.0") + Expect(err).To(HaveOccurred(), "Non-existing VF, no vendor installed") + }) + }) })