From 2ce5a769155fd169bba4da813d493ee9e1ecac6e Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 8 Feb 2024 17:14:11 -0600 Subject: [PATCH] Add RAID Controller interface to enable multiple controllers Introduce a RAIDController interface to abstract linux/system from having to know which raid controller it is interacting with. This also supports handling a raid interface with multiple controllers. - Update build Makefile target to depend on all go files Signed-off-by: Ryan Harper --- Makefile | 2 +- demo/megaraid.go | 3 +- go.mod | 1 + go.sum | 2 + linux/raidcontroller.go | 16 ++++++++ linux/system.go | 49 ++++++++++++------------ linux/util.go | 77 ++++++++++++++++++++++++++++++++++++++ megaraid/linux.go | 83 ----------------------------------------- megaraid/megaraid.go | 11 ++++++ megaraid/storcli.go | 42 +++++++++++++++++++++ 10 files changed, 178 insertions(+), 108 deletions(-) create mode 100644 linux/raidcontroller.go diff --git a/Makefile b/Makefile index 6d4af76..7e072b6 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ all: build check build: .build $(CMDS) -.build: $(GO_FILES) +.build: $(ALL_GO_FILES) go build ./... @touch $@ diff --git a/demo/megaraid.go b/demo/megaraid.go index e9db809..88b22de 100644 --- a/demo/megaraid.go +++ b/demo/megaraid.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/urfave/cli/v2" + "machinerun.io/disko/linux" "machinerun.io/disko/megaraid" ) @@ -69,7 +70,7 @@ func megaraidDiskSummary(c *cli.Context) error { } path := "" - if bname, err := megaraid.NameByDiskID(d.ID); err == nil { + if bname, err := linux.NameByDiskID(mraid.DriverSysfsPath(), d.ID); err == nil { path = "/dev/" + bname } diff --git a/go.mod b/go.mod index 9ff4d53..8d0ce63 100644 --- a/go.mod +++ b/go.mod @@ -12,4 +12,5 @@ require ( github.com/stretchr/testify v1.8.2 github.com/urfave/cli/v2 v2.25.3 golang.org/x/sys v0.8.0 + machinerun.io/disko v0.0.13 // indirect ) diff --git a/go.sum b/go.sum index d1efc79..c06a8a0 100644 --- a/go.sum +++ b/go.sum @@ -634,6 +634,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +machinerun.io/disko v0.0.13 h1:1kkOorxGvLprdTSksh7fOsNK0WZ0Lq+4/ZQgPeuXnsY= +machinerun.io/disko v0.0.13/go.mod h1:qorFih3sB8bmNcHWHyjXujgYfaugOjHj+s2GdK33+lk= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/linux/raidcontroller.go b/linux/raidcontroller.go new file mode 100644 index 0000000..c26fe3c --- /dev/null +++ b/linux/raidcontroller.go @@ -0,0 +1,16 @@ +package linux + +import "machinerun.io/disko" + +type RAIDControllerType string + +const ( + MegaRAIDControllerType RAIDControllerType = "megaraid" +) + +type RAIDController interface { + // Type() RAIDControllerType + GetDiskType(string) (disko.DiskType, error) + IsSysPathRAID(string) bool + DriverSysfsPath() string +} diff --git a/linux/system.go b/linux/system.go index 1274135..6418788 100644 --- a/linux/system.go +++ b/linux/system.go @@ -16,13 +16,15 @@ import ( ) type linuxSystem struct { - megaraid megaraid.MegaRaid + raidctrls []RAIDController } // System returns an linux specific implementation of disko.System interface. func System() disko.System { return &linuxSystem{ - megaraid: megaraid.CachingStorCli(), + raidctrls: []RAIDController{ + megaraid.CachingStorCli(), + }, } } @@ -154,15 +156,22 @@ func (ls *linuxSystem) ScanDisk(devicePath string) (disko.Disk, error) { return disko.Disk{}, err } - diskType, err = ls.getDiskType(devicePath, udInfo) - if err != nil { - return disko.Disk{}, err - } - attachType = getAttachType(udInfo) - if megaraid.IsMegaRaidSysPath(udInfo.Properties["DEVPATH"]) { - attachType = disko.RAID + for _, ctrl := range ls.raidctrls { + if IsSysPathRAID(udInfo.Properties["DEVPATH"], ctrl.DriverSysfsPath()) { + // we know this is device is part of a raid, so if we cannot get + // disk type we must return an error + dType, err := ctrl.GetDiskType(devicePath) + if err != nil { + return disko.Disk{}, fmt.Errorf("failed to get diskType of %q from RAID controller: %s", devicePath, err) + } + + attachType = disko.RAID + diskType = dType + + break + } } ro, err = getDiskReadOnly(name) @@ -277,22 +286,16 @@ func (ls *linuxSystem) Wipe(d disko.Disk) error { return udevSettle() } -func (ls *linuxSystem) getDiskType(path string, udInfo disko.UdevInfo) (disko.DiskType, error) { - ctrl, err := ls.megaraid.Query(0) - if err == nil { - for _, vd := range ctrl.VirtDrives { - if vd.Path == path { - if ctrl.DriveGroups[vd.DriveGroup].IsSSD() { - return disko.SSD, nil - } - - return disko.HDD, nil +func (ls *linuxSystem) GetDiskType(path string, udInfo disko.UdevInfo) (disko.DiskType, error) { + for _, ctrl := range ls.raidctrls { + if IsSysPathRAID(udInfo.Properties["DEVPATH"], ctrl.DriverSysfsPath()) { + dType, err := ctrl.GetDiskType(path) + if err != nil { + return disko.HDD, fmt.Errorf("failed to get diskType of %q from RAID controller: %s", path, err) } + + return dType, nil } - } else if err != megaraid.ErrNoStorcli && err != megaraid.ErrNoController && - err != megaraid.ErrUnsupported { - return disko.HDD, err } - return getDiskType(udInfo) } diff --git a/linux/util.go b/linux/util.go index e9b5e99..23e3577 100644 --- a/linux/util.go +++ b/linux/util.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strconv" "strings" "syscall" @@ -254,3 +255,79 @@ func Floor(val, unit uint64) uint64 { return (val / unit) * unit } + +// IsSysPathRAID - is this sys path (udevadm info's DEVPATH) on a scsi controller. +// +// syspath will look something like +// /devices/pci0000:3a/0000:3a:02.0/0000:3c:00.0/host0/target0:2:2/0:2:2:0/block/sdc +func IsSysPathRAID(syspath string, driverSysPath string) bool { + if !strings.HasPrefix(syspath, "/sys") { + syspath = "/sys" + syspath + } + + if !strings.Contains(syspath, "/host") { + return false + } + + fp, err := filepath.EvalSymlinks(syspath) + if err != nil { + fmt.Printf("seriously? %s\n", err) + return false + } + + for _, path := range GetSysPaths(driverSysPath) { + if strings.HasPrefix(fp, path) { + return true + } + } + + return false +} + +// NameByDiskID - return the linux name (sda) for the disk with given DiskID +func NameByDiskID(driverSysPath string, id int) (string, error) { + // given ID, we expect a single file in: + // /0000:05:00.0/host0/target0:0:/0:0::0/block/ + // Note: This does not work for some controllers such as a MegaRAID SAS3508 + // See https://github.com/project-machine/disko/issues/101 + idStr := fmt.Sprintf("%d", id) + blkDir := driverSysPath + "/*/host*/target0:0:" + idStr + "/0:0:" + idStr + ":0/block/*" + matches, err := filepath.Glob(blkDir) + + if err != nil { + return "", err + } + + if len(matches) != 1 { + return "", fmt.Errorf("found %d matches to %s", len(matches), blkDir) + } + + return path.Base(matches[0]), nil +} + +func GetSysPaths(driverSysPath string) []string { + paths := []string{} + // a raid driver has directory entries for each of the scsi hosts on that controller. + // $cd /sys/bus/pci/drivers/ + // $ for d in *; do [ -d "$d" ] || continue; echo "$d -> $( cd "$d" && pwd -P )"; done + // 0000:3c:00.0 -> /sys/devices/pci0000:3a/0000:3a:02.0/0000:3c:00.0 + // module -> /sys/module/ + + // We take a hack path and consider anything with a ":" in that dir as a host path. + matches, err := filepath.Glob(driverSysPath + "/*:*") + + if err != nil { + fmt.Printf("errors: %s\n", err) + return paths + } + + for _, p := range matches { + fp, err := filepath.EvalSymlinks(p) + + if err == nil { + paths = append(paths, fp) + } + } + + return paths +} diff --git a/megaraid/linux.go b/megaraid/linux.go index e813710..cbd974a 100644 --- a/megaraid/linux.go +++ b/megaraid/linux.go @@ -1,86 +1,3 @@ package megaraid -import ( - "fmt" - "path" - "path/filepath" - "strings" -) - const sysDriverMegaRaidSAS = "/sys/bus/pci/drivers/megaraid_sas" - -// IsMegaRaidSysPath - is this sys path (udevadm info's DEVPATH) on a megaraid controller. -// -// syspath will look something like -// /devices/pci0000:3a/0000:3a:02.0/0000:3c:00.0/host0/target0:2:2/0:2:2:0/block/sdc -func IsMegaRaidSysPath(syspath string) bool { - if !strings.HasPrefix(syspath, "/sys") { - syspath = "/sys" + syspath - } - - if !strings.Contains(syspath, "/host") { - return false - } - - fp, err := filepath.EvalSymlinks(syspath) - if err != nil { - fmt.Printf("seriously? %s\n", err) - return false - } - - for _, path := range getSysPaths() { - if strings.HasPrefix(fp, path) { - return true - } - } - - return false -} - -// NameByDiskID - return the linux name (sda) for the disk with given DiskID -func NameByDiskID(id int) (string, error) { - // given ID, we expect a single file in: - // /0000:05:00.0/host0/target0:0:/0:0::0/block/ - // Note: This does not work for some megaraid controllers such as SAS3508 - // See https://machinerun.io/disko/issues/101 - idStr := fmt.Sprintf("%d", id) - blkDir := sysDriverMegaRaidSAS + "/*/host*/target0:0:" + idStr + "/0:0:" + idStr + ":0/block/*" - matches, err := filepath.Glob(blkDir) - - if err != nil { - return "", err - } - - if len(matches) != 1 { - return "", fmt.Errorf("found %d matches to %s", len(matches), blkDir) - } - - return path.Base(matches[0]), nil -} - -func getSysPaths() []string { - paths := []string{} - // sysDriverMegaRaidSAS has directory entries for each of the scsi hosts on that controller. - // $cd /sys/bus/pci/drivers/megaraid_sas - // $ for d in *; do [ -d "$d" ] || continue; echo "$d -> $( cd "$d" && pwd -P )"; done - // 0000:3c:00.0 -> /sys/devices/pci0000:3a/0000:3a:02.0/0000:3c:00.0 - // module -> /sys/module/megaraid_sas - - // We take a hack path and consider anything with a ":" in that dir as a host path. - matches, err := filepath.Glob(sysDriverMegaRaidSAS + "/*:*") - - if err != nil { - fmt.Printf("errors: %s\n", err) - return paths - } - - for _, p := range matches { - fp, err := filepath.EvalSymlinks(p) - - if err == nil { - paths = append(paths, fp) - } - } - - return paths -} diff --git a/megaraid/megaraid.go b/megaraid/megaraid.go index fa1fc7f..e0d0d2f 100644 --- a/megaraid/megaraid.go +++ b/megaraid/megaraid.go @@ -3,6 +3,8 @@ package megaraid import ( "encoding/json" "errors" + + "machinerun.io/disko" ) // Controller - a Megaraid controller @@ -131,6 +133,15 @@ func (t MediaType) MarshalJSON() ([]byte, error) { type MegaRaid interface { // Query - Query the controller provided Query(int) (Controller, error) + + // GetDiskType - Determine the disk type if controller owns disk + GetDiskType(string) (disko.DiskType, error) + + // DriverSysfsPath - Return the sysfs path to the linux driver for this controller + DriverSysfsPath() string + + // IsSysPathRAID - Check if sysfs path is a device on the controller + IsSysPathRAID(string) bool } // ErrNoController - Error reported by Query if no controller is found. diff --git a/megaraid/storcli.go b/megaraid/storcli.go index da81c9b..88cfdc3 100644 --- a/megaraid/storcli.go +++ b/megaraid/storcli.go @@ -11,6 +11,7 @@ import ( "time" "github.com/patrickmn/go-cache" + "machinerun.io/disko" ) type storCli struct { @@ -74,6 +75,19 @@ func (sc *storCli) Query(cID int) (Controller, error) { return newController(cID, cxDxOut, cxVxOut) } +func (sc *storCli) DriverSysfsPath() string { + return SysfsPCIDriversPath +} + +func (sc *storCli) GetDiskType(path string) (disko.DiskType, error) { + return disko.HDD, fmt.Errorf("missing controller to run query") +} + +// not implemented in driver layer +func (sc *storCli) IsSysPathRAID(syspath string) bool { + return false +} + func newController(cID int, cxDxOut string, cxVxOut string) (Controller, error) { const pathPropName = "OS Drive Name" @@ -607,3 +621,31 @@ func (csc *cachingStorCli) Query(cID int) (Controller, error) { return ctrl, err } + +func (csc *cachingStorCli) GetDiskType(path string) (disko.DiskType, error) { + ctrl, err := csc.Query(0) + if err == nil { + for _, vd := range ctrl.VirtDrives { + if vd.Path == path { + if ctrl.DriveGroups[vd.DriveGroup].IsSSD() { + return disko.SSD, nil + } + + return disko.HDD, nil + } + } + } else if err != ErrNoStorcli && err != ErrNoController && err != ErrUnsupported { + return disko.HDD, err + } + + return disko.HDD, fmt.Errorf("cannot determine disk type") +} + +func (csc *cachingStorCli) DriverSysfsPath() string { + return csc.mr.DriverSysfsPath() +} + +// not implemented in the driver layer +func (csc *cachingStorCli) IsSysPathRAID(syspath string) bool { + return false +}