From e9c80b4283aab7526ed82bfb6868c4a0ba4fcb2d Mon Sep 17 00:00:00 2001 From: Jake Schuurmans Date: Wed, 13 Nov 2024 11:32:25 -0500 Subject: [PATCH 1/5] X13 Inventory support --- providers/supermicro/errors.go | 2 + providers/supermicro/firmware.go | 3 +- providers/supermicro/supermicro.go | 16 +- providers/supermicro/supermicro_test.go | 2 +- providers/supermicro/task.go | 64 ++++++ providers/supermicro/types.go | 9 + providers/supermicro/x11.go | 2 +- providers/supermicro/x12.go | 66 +----- providers/supermicro/x13.go | 273 ++++++++++++++++++++++++ 9 files changed, 364 insertions(+), 73 deletions(-) create mode 100644 providers/supermicro/task.go create mode 100644 providers/supermicro/x13.go diff --git a/providers/supermicro/errors.go b/providers/supermicro/errors.go index c2dcf057..3bda1a30 100644 --- a/providers/supermicro/errors.go +++ b/providers/supermicro/errors.go @@ -12,6 +12,8 @@ var ( ErrXMLAPIUnsupported = errors.New("XML API is unsupported") ErrModelUnknown = errors.New("Model number unknown") ErrModelUnsupported = errors.New("Model not supported") + + ErrUploadTaskIDEmpty = errors.New("firmware upload request returned empty firmware upload verify TaskID") ) type UnexpectedResponseError struct { diff --git a/providers/supermicro/firmware.go b/providers/supermicro/firmware.go index bdbabecc..0c006698 100644 --- a/providers/supermicro/firmware.go +++ b/providers/supermicro/firmware.go @@ -27,6 +27,7 @@ var ( "X11SSE-F", "X12STH-SYS", "X12SPO-NTF", + "X13DEM", } errUploadTaskIDExpected = errors.New("expected an firmware upload taskID") @@ -46,7 +47,7 @@ func (c *Client) FirmwareUpload(ctx context.Context, component string, file *os. return "", err } - // // expect atleast 5 minutes left in the deadline to proceed with the upload + // expect atleast 5 minutes left in the deadline to proceed with the upload d, _ := ctx.Deadline() if time.Until(d) < 5*time.Minute { return "", errors.New("remaining context deadline insufficient to perform update: " + time.Until(d).String()) diff --git a/providers/supermicro/supermicro.go b/providers/supermicro/supermicro.go index 974dd6b1..bff35d82 100644 --- a/providers/supermicro/supermicro.go +++ b/providers/supermicro/supermicro.go @@ -182,8 +182,7 @@ func (c *Client) Open(ctx context.Context) (err error) { return err } - if !bytes.Contains(body, []byte(`url_redirect.cgi?url_name=mainmenu`)) && - !bytes.Contains(body, []byte(`url_redirect.cgi?url_name=topmenu`)) { + if !bytes.Contains(body, []byte(`url_redirect.cgi?url_name=topmenu`)) { return closeWithError(ctx, errors.Wrap(bmclibErrs.ErrLoginFailed, "unexpected response contents")) } @@ -290,10 +289,11 @@ func (c *Client) ResetBiosConfiguration(ctx context.Context) (err error) { func (c *Client) bmcQueryor(ctx context.Context) (bmcQueryor, error) { x11 := newX11Client(c.serviceClient, c.log) x12 := newX12Client(c.serviceClient, c.log) + x13 := newX13Client(c.serviceClient, c.log) var queryor bmcQueryor - for _, bmc := range []bmcQueryor{x11, x12} { + for _, bmc := range []bmcQueryor{x11, x12, x13} { var err error // Note to maintainers: x12 lacks support for the ipmi.cgi endpoint, @@ -316,11 +316,15 @@ func (c *Client) bmcQueryor(ctx context.Context) (bmcQueryor, error) { } model := strings.ToLower(queryor.deviceModel()) - if !strings.HasPrefix(model, "x12") && !strings.HasPrefix(model, "x11") { - return nil, errors.Wrap(ErrModelUnsupported, "expected one of X11* or X12*, got:"+model) + acceptedModels := []string{"x11", "x12", "x13"} + + for _, acceptedModel := range acceptedModels { + if strings.HasPrefix(model, acceptedModel) { + return queryor, nil + } } - return queryor, nil + return nil, errors.Wrapf(ErrModelUnsupported, "Got: %s, expected one of: %s", model, strings.Join(acceptedModels, ", ")) } func parseToken(body []byte) string { diff --git a/providers/supermicro/supermicro_test.go b/providers/supermicro/supermicro_test.go index 135408e9..066ad972 100644 --- a/providers/supermicro/supermicro_test.go +++ b/providers/supermicro/supermicro_test.go @@ -143,7 +143,7 @@ func TestOpen(t *testing.T) { diff --git a/providers/supermicro/task.go b/providers/supermicro/task.go new file mode 100644 index 00000000..a0ed0b31 --- /dev/null +++ b/providers/supermicro/task.go @@ -0,0 +1,64 @@ +package supermicro + +import ( + "fmt" + "strings" + + common "github.com/metal-toolbox/bmc-common" + "github.com/pkg/errors" + "github.com/stmcginnis/gofish/redfish" + "golang.org/x/exp/slices" +) + +// noTasksRunning returns an error if a firmware related task was found active +func noTasksRunning(component string, t *redfish.Task) error { + if t.TaskState == "Killed" { + return nil + } + + errTaskActive := errors.New("A firmware task was found active for component: " + component) + + const ( + // The redfish task name when the BMC is verifies the uploaded BMC firmware. + verifyBMCFirmware = "BMC Verify" + // The redfish task name when the BMC is installing the uploaded BMC firmware. + updateBMCFirmware = "BMC Update" + // The redfish task name when the BMC is verifies the uploaded BIOS firmware. + verifyBIOSFirmware = "BIOS Verify" + // The redfish task name when the BMC is installing the uploaded BIOS firmware. + updateBIOSFirmware = "BIOS Update" + ) + + var verifyTaskName, updateTaskName string + + switch strings.ToUpper(component) { + case common.SlugBMC: + verifyTaskName = verifyBMCFirmware + updateTaskName = updateBMCFirmware + case common.SlugBIOS: + verifyTaskName = verifyBIOSFirmware + updateTaskName = updateBIOSFirmware + } + + taskInfo := fmt.Sprintf("id: %s, state: %s, status: %s", t.ID, t.TaskState, t.TaskStatus) + + switch t.Name { + case verifyTaskName: + return errors.Wrap(errTaskActive, taskInfo) + case updateTaskName: + return errors.Wrap(errTaskActive, taskInfo) + default: + return nil + } +} + +func stateFinalized(s redfish.TaskState) bool { + finalized := []redfish.TaskState{ + redfish.CompletedTaskState, + redfish.CancelledTaskState, + redfish.InterruptedTaskState, + redfish.ExceptionTaskState, + } + + return slices.Contains(finalized, s) +} diff --git a/providers/supermicro/types.go b/providers/supermicro/types.go index 3946b9a4..d1214028 100644 --- a/providers/supermicro/types.go +++ b/providers/supermicro/types.go @@ -16,3 +16,12 @@ type Board struct { ProdName string `xml:"PROD_NAME,attr"` SerialNum string `xml:"SERIAL_NUM,attr"` } + +type Supermicro struct { + BIOS map[string]bool `json:"BIOS,omitempty"` + BMC map[string]bool `json:"BMC,omitempty"` +} + +type OEM struct { + Supermicro `json:"Supermicro"` +} diff --git a/providers/supermicro/x11.go b/providers/supermicro/x11.go index 112bcb05..e1e64c04 100644 --- a/providers/supermicro/x11.go +++ b/providers/supermicro/x11.go @@ -39,7 +39,7 @@ func (c *x11) queryDeviceModel(ctx context.Context) (string, error) { errBoardPartNumUnknown := errors.New("baseboard part number unknown") data, err := c.fruInfo(ctx) if err != nil { - if strings.Contains(err.Error(), "404") { + if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "") { return "", ErrXMLAPIUnsupported } diff --git a/providers/supermicro/x12.go b/providers/supermicro/x12.go index baa5dfa2..02d1d9a6 100644 --- a/providers/supermicro/x12.go +++ b/providers/supermicro/x12.go @@ -53,10 +53,6 @@ func (c *x12) queryDeviceModel(ctx context.Context) (string, error) { return c.model, nil } -var ( - errUploadTaskIDEmpty = errors.New("firmware upload request returned empty firmware upload verify TaskID") -) - func (c *x12) supportsInstall(component string) error { errComponentNotSupported := fmt.Errorf("component %s on device %s not supported", component, c.model) @@ -112,7 +108,7 @@ func (c *x12) firmwareUpload(ctx context.Context, component string, file *os.Fil } if taskID == "" { - return "", errUploadTaskIDEmpty + return "", ErrUploadTaskIDEmpty } return taskID, nil @@ -140,64 +136,6 @@ func (c *x12) firmwareTaskActive(ctx context.Context, component string) error { return nil } -// noTasksRunning returns an error if a firmware related task was found active -func noTasksRunning(component string, t *redfish.Task) error { - errTaskActive := errors.New("A firmware task was found active for component: " + component) - - const ( - // The redfish task name when the BMC is verifies the uploaded BMC firmware. - verifyBMCFirmware = "BMC Verify" - // The redfish task name when the BMC is installing the uploaded BMC firmware. - updateBMCFirmware = "BMC Update" - // The redfish task name when the BMC is verifies the uploaded BIOS firmware. - verifyBIOSFirmware = "BIOS Verify" - // The redfish task name when the BMC is installing the uploaded BIOS firmware. - updateBIOSFirmware = "BIOS Update" - ) - - var verifyTaskName, updateTaskName string - - switch strings.ToUpper(component) { - case common.SlugBMC: - verifyTaskName = verifyBMCFirmware - updateTaskName = updateBMCFirmware - case common.SlugBIOS: - verifyTaskName = verifyBIOSFirmware - updateTaskName = updateBIOSFirmware - } - - taskInfo := fmt.Sprintf("id: %s, state: %s, status: %s", t.ID, t.TaskState, t.TaskStatus) - - switch t.Name { - case verifyTaskName: - return errors.Wrap(errTaskActive, taskInfo) - case updateTaskName: - return errors.Wrap(errTaskActive, taskInfo) - default: - return nil - } -} - -func stateFinalized(s redfish.TaskState) bool { - finalized := []redfish.TaskState{ - redfish.CompletedTaskState, - redfish.CancelledTaskState, - redfish.InterruptedTaskState, - redfish.ExceptionTaskState, - } - - return slices.Contains(finalized, s) -} - -type Supermicro struct { - BIOS map[string]bool `json:"BIOS,omitempty"` - BMC map[string]bool `json:"BMC,omitempty"` -} - -type OEM struct { - Supermicro `json:"Supermicro"` -} - // redfish OEM fw install parameters func (c *x12) biosFwInstallParams() (map[string]bool, error) { switch c.model { @@ -222,7 +160,7 @@ func (c *x12) biosFwInstallParams() (map[string]bool, error) { }, nil default: // ideally we never get in this position, since theres model number validation in parent callers. - return nil, errors.New("unsupported model for BIOS fw install: " + c.model) + return nil, errors.New("unsupported model for X12 BIOS fw install: " + c.model) } } diff --git a/providers/supermicro/x13.go b/providers/supermicro/x13.go new file mode 100644 index 00000000..5d6388cf --- /dev/null +++ b/providers/supermicro/x13.go @@ -0,0 +1,273 @@ +package supermicro + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/go-logr/logr" + common "github.com/metal-toolbox/bmc-common" + "github.com/metal-toolbox/bmclib/constants" + brrs "github.com/metal-toolbox/bmclib/errors" + rfw "github.com/metal-toolbox/bmclib/internal/redfishwrapper" + "github.com/pkg/errors" + "github.com/stmcginnis/gofish/redfish" + "golang.org/x/exp/slices" +) + +type x13 struct { + *serviceClient + model string + log logr.Logger +} + +func newX13Client(client *serviceClient, logger logr.Logger) bmcQueryor { + return &x13{ + serviceClient: client, + log: logger, + } +} + +func (c *x13) deviceModel() string { + return c.model +} + +func (c *x13) queryDeviceModel(ctx context.Context) (string, error) { + if err := c.redfishSession(ctx); err != nil { + return "", err + } + + _, model, err := c.redfish.DeviceVendorModel(ctx) + if err != nil { + return "", err + } + + if model == "" { + return "", errors.Wrap(ErrModelUnknown, "empty value") + } + + c.model = common.FormatProductName(model) + + return c.model, nil +} + +func (c *x13) supportsInstall(component string) error { + errComponentNotSupported := fmt.Errorf("component %s on device %s not supported", component, c.model) + + supported := []string{common.SlugBIOS, common.SlugBMC} + if !slices.Contains(supported, strings.ToUpper(component)) { + return errComponentNotSupported + } + + return nil +} + +func (c *x13) firmwareInstallSteps(component string) ([]constants.FirmwareInstallStep, error) { + if err := c.supportsInstall(component); err != nil { + return nil, err + } + + return []constants.FirmwareInstallStep{ + constants.FirmwareInstallStepUpload, + constants.FirmwareInstallStepUploadStatus, + constants.FirmwareInstallStepInstallUploaded, + constants.FirmwareInstallStepInstallStatus, + }, nil +} + +// upload firmware +func (c *x13) firmwareUpload(ctx context.Context, component string, file *os.File) (taskID string, err error) { + if err = c.supportsInstall(component); err != nil { + return "", err + } + + err = c.firmwareTaskActive(ctx, component) + if err != nil { + return "", err + } + + targetID, err := c.redfishOdataID(ctx, component) + if err != nil { + return "", err + } + + params, err := c.redfishParameters(component, targetID) + if err != nil { + return "", err + } + + taskID, err = c.redfish.FirmwareUpload(ctx, file, params) + if err != nil { + if strings.Contains(err.Error(), "OemFirmwareAlreadyInUpdateMode") { + return "", errors.Wrap(brrs.ErrBMCColdResetRequired, "BMC currently in update mode, either continue the update OR if no update is currently running - reset the BMC") + } + + return "", errors.Wrap(err, "error in firmware upload") + } + + if taskID == "" { + return "", ErrUploadTaskIDEmpty + } + + return taskID, nil +} + +// returns an error when a bmc firmware install is active +func (c *x13) firmwareTaskActive(ctx context.Context, component string) error { + tasks, err := c.redfish.Tasks(ctx) + if err != nil { + return errors.Wrap(err, "error querying redfish tasks") + } + + for _, t := range tasks { + t := t + + if stateFinalized(t.TaskState) { + continue + } + + if err := noTasksRunning(component, t); err != nil { + return err + } + } + + return nil +} + +// redfish OEM fw install parameters +func (c *x13) biosFwInstallParams() (map[string]bool, error) { + switch c.model { + case "x12spo-ntf": + return map[string]bool{ + "PreserveME": false, + "PreserveNVRAM": false, + "PreserveSMBIOS": true, + "BackupBIOS": false, + "PreserveBOOTCONF": true, + }, nil + case "x12sth-sys": + return map[string]bool{ + "PreserveME": false, + "PreserveNVRAM": false, + "PreserveSMBIOS": true, + "PreserveOA": true, + "PreserveSETUPCONF": true, + "PreserveSETUPPWD": true, + "PreserveSECBOOTKEY": true, + "PreserveBOOTCONF": true, + }, nil + default: + // ideally we never get in this position, since theres model number validation in parent callers. + return nil, errors.New("unsupported model for X13 BIOS fw install: " + c.model) + } +} + +// redfish OEM fw install parameters +func (c *x13) bmcFwInstallParams() map[string]bool { + return map[string]bool{ + "PreserveCfg": true, + "PreserveSdr": true, + "PreserveSsl": true, + } +} + +func (c *x13) redfishParameters(component, targetODataID string) (*rfw.RedfishUpdateServiceParameters, error) { + errUnsupported := errors.New("redfish parameters for x13 hardware component not supported: " + component) + + oem := OEM{} + + biosInstallParams, err := c.biosFwInstallParams() + if err != nil { + return nil, err + } + + switch strings.ToUpper(component) { + case common.SlugBIOS: + oem.Supermicro.BIOS = biosInstallParams + case common.SlugBMC: + oem.Supermicro.BMC = c.bmcFwInstallParams() + default: + return nil, errUnsupported + } + + b, err := json.Marshal(oem) + if err != nil { + return nil, errors.Wrap(err, "error preparing redfish parameters") + } + + return &rfw.RedfishUpdateServiceParameters{ + // NOTE: + // X13s support the OnReset Apply time for BIOS updates if we want to implement that in the future. + OperationApplyTime: constants.OnStartUpdateRequest, + Targets: []string{targetODataID}, + Oem: b, + }, nil +} + +func (c *x13) redfishOdataID(ctx context.Context, component string) (string, error) { + errUnsupported := errors.New("unable to return redfish OData ID for unsupported component: " + component) + + switch strings.ToUpper(component) { + case common.SlugBMC: + return c.redfish.ManagerOdataID(ctx) + case common.SlugBIOS: + // hardcoded since SMCs without the DCMS license will throw license errors + return "/redfish/v1/Systems/1/Bios", nil + //return c.redfish.SystemsBIOSOdataID(ctx) + } + + return "", errUnsupported +} + +func (c *x13) firmwareInstallUploaded(ctx context.Context, component, uploadTaskID string) (installTaskID string, err error) { + if err = c.supportsInstall(component); err != nil { + return "", err + } + + task, err := c.redfish.Task(ctx, uploadTaskID) + if err != nil { + e := fmt.Sprintf("error querying redfish tasks for firmware upload taskID: %s, err: %s", uploadTaskID, err.Error()) + return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, e) + } + + taskInfo := fmt.Sprintf("id: %s, state: %s, status: %s", task.ID, task.TaskState, task.TaskStatus) + + if task.TaskState != redfish.CompletedTaskState { + return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, taskInfo) + } + + if task.TaskStatus != "OK" { + return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, taskInfo) + } + + return c.redfish.StartUpdateForUploadedFirmware(ctx) +} + +func (c *x13) firmwareTaskStatus(ctx context.Context, component, taskID string) (state constants.TaskState, status string, err error) { + if err = c.supportsInstall(component); err != nil { + return "", "", errors.Wrap(brrs.ErrFirmwareTaskStatus, err.Error()) + } + + return c.redfish.TaskStatus(ctx, taskID) +} + +func (c *x13) getBootProgress() (*redfish.BootProgress, error) { + bps, err := c.redfish.GetBootProgress() + if err != nil { + return nil, err + } + return bps[0], nil +} + +// this is some syntactic sugar to avoid having to code potentially provider- or model-specific knowledge into a caller +func (c *x13) bootComplete() (bool, error) { + bp, err := c.getBootProgress() + if err != nil { + return false, err + } + // TODO? + // we determined this by experiment on X12STH-SYS with redfish 1.14.0 + return bp.LastState == redfish.SystemHardwareInitializationCompleteBootProgressTypes, nil +} From 9ab35be958b624518c2edebcc992e8455a809408 Mon Sep 17 00:00:00 2001 From: Jake Schuurmans Date: Mon, 18 Nov 2024 16:18:20 -0500 Subject: [PATCH 2/5] X13 Firmware update support --- examples/upload-install-firmware/main.go | 198 +++++++++++++++++++++++ go.mod | 4 +- go.sum | 2 - internal/redfishwrapper/firmware.go | 15 ++ providers/supermicro/supermicro.go | 52 +++--- providers/supermicro/x13.go | 26 ++- 6 files changed, 256 insertions(+), 41 deletions(-) create mode 100644 examples/upload-install-firmware/main.go diff --git a/examples/upload-install-firmware/main.go b/examples/upload-install-firmware/main.go new file mode 100644 index 00000000..d68e13b4 --- /dev/null +++ b/examples/upload-install-firmware/main.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "crypto/x509" + "errors" + "flag" + "io/ioutil" + "log" + "os" + "strings" + "time" + + "github.com/bombsimon/logrusr/v2" + bmclib "github.com/metal-toolbox/bmclib" + "github.com/metal-toolbox/bmclib/constants" + bmclibErrs "github.com/metal-toolbox/bmclib/errors" + "github.com/sirupsen/logrus" +) + +func main() { + user := flag.String("user", "", "Username to login with") + pass := flag.String("password", "", "Username to login with") + host := flag.String("host", "", "BMC hostname to connect to") + component := flag.String("component", "", "Component to be updated (bmc, bios.. etc)") + withSecureTLS := flag.Bool("secure-tls", false, "Enable secure TLS") + certPoolPath := flag.String("cert-pool", "", "Path to an file containing x509 CAs. An empty string uses the system CAs. Only takes effect when --secure-tls=true") + firmwarePath := flag.String("firmware", "", "The local path of the firmware to install") + firmwareVersion := flag.String("version", "", "The firmware version being installed") + + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + l := logrus.New() + l.Level = logrus.TraceLevel + logger := logrusr.New(l) + + if *host == "" || *user == "" || *pass == "" { + l.Fatal("required host/user/pass parameters not defined") + } + + if *component == "" { + l.Fatal("component parameter required (must be a component slug - bmc, bios etc)") + } + + clientOpts := []bmclib.Option{ + bmclib.WithLogger(logger), + bmclib.WithPerProviderTimeout(time.Minute * 30), + } + + if *withSecureTLS { + var pool *x509.CertPool + if *certPoolPath != "" { + pool = x509.NewCertPool() + data, err := ioutil.ReadFile(*certPoolPath) + if err != nil { + l.Fatal(err) + } + pool.AppendCertsFromPEM(data) + } + // a nil pool uses the system certs + clientOpts = append(clientOpts, bmclib.WithSecureTLS(pool)) + } + + cl := bmclib.NewClient(*host, *user, *pass, clientOpts...) + err := cl.Open(ctx) + if err != nil { + l.Fatal(err, "bmc login failed") + } + + defer cl.Close(ctx) + + // open file handle + fh, err := os.Open(*firmwarePath) + if err != nil { + l.Fatal(err) + } + defer fh.Close() + + steps, err := cl.FirmwareInstallSteps(ctx, *component) + if err != nil { + l.Fatal(err) + } + + l.Infof("Steps: %+v", steps) + + taskID := "" + var lastStep constants.FirmwareInstallStep = "" + for _, step := range steps { + l.Infof("Step: %s", step) + + switch step { + case constants.FirmwareInstallStepUploadInitiateInstall: + taskID, err = cl.FirmwareInstallUploadAndInitiate(ctx, *component, fh) + if err != nil { + l.Fatal(err) + } + case constants.FirmwareInstallStepInstallStatus: + fallthrough + case constants.FirmwareInstallStepUploadStatus: + if taskID == "" { + l.Fatal("taskID wasnt set") + } + if lastStep == "" { + l.Fatal("lastStep wasnt set") + } + firmwareInstallStatusWait(ctx, cl, l, lastStep, *component, *firmwareVersion, taskID) + case constants.FirmwareInstallStepUpload: + taskID, err = cl.FirmwareUpload(ctx, *component, fh) + if err != nil { + l.Fatal(err) + } + case constants.FirmwareInstallStepInstallUploaded: + if taskID == "" { + l.Fatal("taskID wasnt set") + } + taskID, err = cl.FirmwareInstallUploaded(ctx, *component, taskID) + if err != nil { + l.Fatal(err) + } + case constants.FirmwareInstallStepPowerOffHost: + _, err = cl.SetPowerState(ctx, "off") + if err != nil { + l.Fatal(err) + } + case constants.FirmwareInstallStepResetBMCPostInstall: + fallthrough + case constants.FirmwareInstallStepResetBMCOnInstallFailure: + _, err = cl.ResetBMC(ctx, "GracefulRestart") + if err != nil { + l.Fatal(err) + } + default: + l.Fatal("unknown firmware install step") + } + + lastStep = step + } +} + +func firmwareInstallStatusWait(ctx context.Context, cl *bmclib.Client, l *logrus.Logger, step constants.FirmwareInstallStep, component, firmwareVersion, taskID string) { + for range 300 { + if ctx.Err() != nil { + l.Fatal(ctx.Err()) + } + + state, status, err := cl.FirmwareTaskStatus(ctx, step, component, taskID, firmwareVersion) + if err != nil { + // when its under update a connection refused is returned + if strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "operation timed out") { + l.Info("BMC refused connection, BMC most likely resetting...") + time.Sleep(2 * time.Second) + + continue + } + + if errors.Is(err, bmclibErrs.ErrSessionExpired) || strings.Contains(err.Error(), "session expired") { + err := cl.Open(ctx) + if err != nil { + l.Fatal(err, "bmc re-login failed") + } + + l.WithFields(logrus.Fields{"state": state, "component": component}).Info("BMC session expired, logging in...") + + continue + } + + log.Fatal(err) + } + + switch state { + case constants.FirmwareInstallRunning, constants.FirmwareInstallInitializing: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s running", step) + case constants.FirmwareInstallFailed: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s failed", step) + os.Exit(1) + case constants.FirmwareInstallComplete, constants.FirmwareInstallQueued: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s completed", step) + return + case constants.FirmwareInstallPowerCycleHost: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("host powercycle required") + + if _, err := cl.SetPowerState(ctx, "cycle"); err != nil { + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("error power cycling host for %s", step) + os.Exit(1) + } + + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("host power cycled, all done!") + return + default: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("unknown state returned") + } + + time.Sleep(2 * time.Second) + } +} diff --git a/go.mod b/go.mod index 573dcd99..7a9254e3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/metal-toolbox/bmclib -go 1.21 +go 1.22 require ( dario.cat/mergo v1.0.0 @@ -13,6 +13,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef github.com/jacobweinstock/registrar v0.4.7 + github.com/metal-toolbox/bmc-common v1.0.2 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 github.com/sirupsen/logrus v1.9.3 @@ -34,7 +35,6 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/metal-toolbox/bmc-common v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/satori/go.uuid v1.2.0 // indirect golang.org/x/sys v0.20.0 // indirect diff --git a/go.sum b/go.sum index 87ec3331..13ccddb6 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 h1:t95Grn2 github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230/go.mod h1:t2EzW1qybnPDQ3LR/GgeF0GOzHUXT5IVMLP2gkW1cmc= github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 h1:a0MBqYm44o0NcthLKCljZHe1mxlN6oahCQHHThnSwB4= github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22/go.mod h1:/B7V22rcz4860iDqstGvia/2+IYWXf3/JdQCVd/1D2A= -github.com/bmc-toolbox/common v0.0.0-20240806132831-ba8adc6a35e3 h1:/BjZSX/sphptIdxpYo4wxAQkgMLyMMgfdl48J9DKNeE= -github.com/bmc-toolbox/common v0.0.0-20240806132831-ba8adc6a35e3/go.mod h1:Cdnkm+edb6C0pVkyCrwh3JTXAe0iUF9diDG/DztPI9I= github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM= github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2lc5YJvzmOJio= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= diff --git a/internal/redfishwrapper/firmware.go b/internal/redfishwrapper/firmware.go index 287cda17..44523e13 100644 --- a/internal/redfishwrapper/firmware.go +++ b/internal/redfishwrapper/firmware.go @@ -169,6 +169,21 @@ func (c *Client) StartUpdateForUploadedFirmware(ctx context.Context) (taskID str return taskIDFromResponseBody(response) } +// StartUpdateForUploadedFirmware starts an update for a firmware file previously uploaded +func (c *Client) StartUpdateForUploadedFirmwareNoTaskID(ctx context.Context) error { + updateService, err := c.client.Service.UpdateService() + if err != nil { + return errors.Wrap(err, "error querying redfish update service") + } + + err = updateService.StartUpdate() + if err != nil { + return errors.Wrap(err, "error querying redfish start update endpoint") + } + + return nil +} + type TaskAccepted struct { Accepted struct { Code string `json:"code"` diff --git a/providers/supermicro/supermicro.go b/providers/supermicro/supermicro.go index bff35d82..01602617 100644 --- a/providers/supermicro/supermicro.go +++ b/providers/supermicro/supermicro.go @@ -287,18 +287,31 @@ func (c *Client) ResetBiosConfiguration(ctx context.Context) (err error) { } func (c *Client) bmcQueryor(ctx context.Context) (bmcQueryor, error) { - x11 := newX11Client(c.serviceClient, c.log) - x12 := newX12Client(c.serviceClient, c.log) - x13 := newX13Client(c.serviceClient, c.log) - - var queryor bmcQueryor - - for _, bmc := range []bmcQueryor{x11, x12, x13} { + bmcModels := []struct { + bmc bmcQueryor + modelFamily string + }{ + { + newX11Client(c.serviceClient, c.log), + "x11", + }, + { + newX12Client(c.serviceClient, c.log), + "x12", + }, + { + newX13Client(c.serviceClient, c.log), + "x13", + }, + } + + var model string + for _, bmcModel := range bmcModels { var err error - // Note to maintainers: x12 lacks support for the ipmi.cgi endpoint, + // Note to maintainers: x12 and x13 lacks support for the ipmi.cgi endpoint, // which will lead to our graceful handling of ErrXMLAPIUnsupported below. - _, err = bmc.queryDeviceModel(ctx) + tempModel, err := bmcModel.bmc.queryDeviceModel(ctx) if err != nil { if errors.Is(err, ErrXMLAPIUnsupported) { continue @@ -307,24 +320,17 @@ func (c *Client) bmcQueryor(ctx context.Context) (bmcQueryor, error) { return nil, errors.Wrap(ErrModelUnknown, err.Error()) } - queryor = bmc - break - } - - if queryor == nil { - return nil, errors.Wrap(ErrModelUnknown, "failed to setup query client") - } - - model := strings.ToLower(queryor.deviceModel()) - acceptedModels := []string{"x11", "x12", "x13"} + if strings.HasPrefix(strings.ToLower(tempModel), bmcModel.modelFamily) { + return bmcModel.bmc, nil + } - for _, acceptedModel := range acceptedModels { - if strings.HasPrefix(model, acceptedModel) { - return queryor, nil + // For returning more informative error bellow + if tempModel != "" { + model = tempModel } } - return nil, errors.Wrapf(ErrModelUnsupported, "Got: %s, expected one of: %s", model, strings.Join(acceptedModels, ", ")) + return nil, errors.Wrapf(ErrModelUnknown, "failed to setup query client, had unsupported model: %s", model) } func parseToken(body []byte) string { diff --git a/providers/supermicro/x13.go b/providers/supermicro/x13.go index 5d6388cf..0df7d08e 100644 --- a/providers/supermicro/x13.go +++ b/providers/supermicro/x13.go @@ -139,24 +139,15 @@ func (c *x13) firmwareTaskActive(ctx context.Context, component string) error { // redfish OEM fw install parameters func (c *x13) biosFwInstallParams() (map[string]bool, error) { switch c.model { - case "x12spo-ntf": + case "x13dem": return map[string]bool{ - "PreserveME": false, - "PreserveNVRAM": false, - "PreserveSMBIOS": true, - "BackupBIOS": false, - "PreserveBOOTCONF": true, - }, nil - case "x12sth-sys": - return map[string]bool{ - "PreserveME": false, - "PreserveNVRAM": false, "PreserveSMBIOS": true, "PreserveOA": true, "PreserveSETUPCONF": true, "PreserveSETUPPWD": true, "PreserveSECBOOTKEY": true, "PreserveBOOTCONF": true, + "BackupBIOS": false, }, nil default: // ideally we never get in this position, since theres model number validation in parent callers. @@ -170,6 +161,7 @@ func (c *x13) bmcFwInstallParams() map[string]bool { "PreserveCfg": true, "PreserveSdr": true, "PreserveSsl": true, + "BackupBMC": false, } } @@ -234,7 +226,7 @@ func (c *x13) firmwareInstallUploaded(ctx context.Context, component, uploadTask taskInfo := fmt.Sprintf("id: %s, state: %s, status: %s", task.ID, task.TaskState, task.TaskStatus) - if task.TaskState != redfish.CompletedTaskState { + if task.TaskState != redfish.CompletedTaskState && task.TaskState != redfish.PendingTaskState { return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, taskInfo) } @@ -242,7 +234,13 @@ func (c *x13) firmwareInstallUploaded(ctx context.Context, component, uploadTask return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, taskInfo) } - return c.redfish.StartUpdateForUploadedFirmware(ctx) + err = c.redfish.StartUpdateForUploadedFirmwareNoTaskID(ctx) + if err != nil { + return "", err + } + + // X13s dont create a new task id when going from upload to install, so we pass through the same one + return uploadTaskID, nil } func (c *x13) firmwareTaskStatus(ctx context.Context, component, taskID string) (state constants.TaskState, status string, err error) { @@ -267,7 +265,7 @@ func (c *x13) bootComplete() (bool, error) { if err != nil { return false, err } - // TODO? + // we determined this by experiment on X12STH-SYS with redfish 1.14.0 return bp.LastState == redfish.SystemHardwareInitializationCompleteBootProgressTypes, nil } From b7c01a0586a6546f4262a4b12a777b0eeb46b809 Mon Sep 17 00:00:00 2001 From: Jake Schuurmans Date: Tue, 3 Dec 2024 13:02:45 -0500 Subject: [PATCH 3/5] Fix up examples --- examples/reset_bmc/{reset_bmc.go => main.go} | 14 ++++++------- examples/upload-install-firmware/main.go | 22 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 10 deletions(-) rename examples/reset_bmc/{reset_bmc.go => main.go} (69%) diff --git a/examples/reset_bmc/reset_bmc.go b/examples/reset_bmc/main.go similarity index 69% rename from examples/reset_bmc/reset_bmc.go rename to examples/reset_bmc/main.go index 740d8b48..1379381f 100644 --- a/examples/reset_bmc/reset_bmc.go +++ b/examples/reset_bmc/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "flag" "log" "os" "time" @@ -16,22 +17,19 @@ func main() { defer cancel() // set BMC parameters here - host := "10.211.132.157" - user := "root" - pass := "yxvZdxAQ38ZWlZ" + host := flag.String("host", "", "BMC hostname to connect to") + user := flag.String("user", "", "Username to login with") + pass := flag.String("pass", "", "Username to login with") + flag.Parse() l := logrus.New() l.Level = logrus.DebugLevel logger := logrusr.New(l) - if host == "" || user == "" || pass == "" { - log.Fatal("required host/user/pass parameters not defined") - } - os.Setenv("DEBUG_BMCLIB", "true") defer os.Unsetenv("DEBUG_BMCLIB") - cl := bmclib.NewClient(host, user, pass, bmclib.WithLogger(logger)) + cl := bmclib.NewClient(*host, *user, *pass, bmclib.WithLogger(logger)) err := cl.Open(ctx) if err != nil { diff --git a/examples/upload-install-firmware/main.go b/examples/upload-install-firmware/main.go index d68e13b4..988a2536 100644 --- a/examples/upload-install-firmware/main.go +++ b/examples/upload-install-firmware/main.go @@ -5,6 +5,7 @@ import ( "crypto/x509" "errors" "flag" + "fmt" "io/ioutil" "log" "os" @@ -65,6 +66,7 @@ func main() { } cl := bmclib.NewClient(*host, *user, *pass, clientOpts...) + err := cl.Open(ctx) if err != nil { l.Fatal(err, "bmc login failed") @@ -84,7 +86,11 @@ func main() { l.Fatal(err) } - l.Infof("Steps: %+v", steps) + sprinted := fmt.Sprintf("%v", steps) + trimmed := strings.Trim(sprinted, "[]") + replaced := strings.Replace(trimmed, " ", " - ", 0) + + l.Infof("Steps: %s", replaced) taskID := "" var lastStep constants.FirmwareInstallStep = "" @@ -97,11 +103,15 @@ func main() { if err != nil { l.Fatal(err) } + // X11 doesnt have a taskID, so lets give it a dummy one + if taskID == "" { + taskID = "0" + } case constants.FirmwareInstallStepInstallStatus: fallthrough case constants.FirmwareInstallStepUploadStatus: if taskID == "" { - l.Fatal("taskID wasnt set") + l.Warn("taskID wasnt set, continueing anyway") } if lastStep == "" { l.Fatal("lastStep wasnt set") @@ -112,6 +122,10 @@ func main() { if err != nil { l.Fatal(err) } + // X11 doesnt have a taskID, so lets give it a dummy one + if taskID == "" { + taskID = "0" + } case constants.FirmwareInstallStepInstallUploaded: if taskID == "" { l.Fatal("taskID wasnt set") @@ -120,6 +134,10 @@ func main() { if err != nil { l.Fatal(err) } + // X11 doesnt have a taskID, so lets give it a dummy one + if taskID == "" { + taskID = "0" + } case constants.FirmwareInstallStepPowerOffHost: _, err = cl.SetPowerState(ctx, "off") if err != nil { From 60c464defce281a9bbaec3469ce239cd66f1f565 Mon Sep 17 00:00:00 2001 From: Jake Schuurmans Date: Tue, 3 Dec 2024 13:03:03 -0500 Subject: [PATCH 4/5] remove duplicate import --- bmc/firmware.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bmc/firmware.go b/bmc/firmware.go index 09c90528..524f47b8 100644 --- a/bmc/firmware.go +++ b/bmc/firmware.go @@ -7,7 +7,6 @@ import ( "os" "github.com/metal-toolbox/bmclib/constants" - bconsts "github.com/metal-toolbox/bmclib/constants" bmclibErrs "github.com/metal-toolbox/bmclib/errors" "github.com/hashicorp/go-multierror" @@ -481,7 +480,7 @@ type FirmwareTaskVerifier interface { // return values: // state - returns one of the FirmwareTask statuses (see devices/constants.go). // status - returns firmware task progress or other arbitrary task information. - FirmwareTaskStatus(ctx context.Context, kind bconsts.FirmwareInstallStep, component, taskID, installVersion string) (state constants.TaskState, status string, err error) + FirmwareTaskStatus(ctx context.Context, kind constants.FirmwareInstallStep, component, taskID, installVersion string) (state constants.TaskState, status string, err error) } // firmwareTaskVerifierProvider is an internal struct to correlate an implementation/provider and its name @@ -492,7 +491,7 @@ type firmwareTaskVerifierProvider struct { // firmwareTaskStatus returns the status of the firmware upload process. -func firmwareTaskStatus(ctx context.Context, kind bconsts.FirmwareInstallStep, component, taskID, installVersion string, generic []firmwareTaskVerifierProvider) (state constants.TaskState, status string, metadata Metadata, err error) { +func firmwareTaskStatus(ctx context.Context, kind constants.FirmwareInstallStep, component, taskID, installVersion string, generic []firmwareTaskVerifierProvider) (state constants.TaskState, status string, metadata Metadata, err error) { metadata = newMetadata() for _, elem := range generic { @@ -522,7 +521,7 @@ func firmwareTaskStatus(ctx context.Context, kind bconsts.FirmwareInstallStep, c } // FirmwareTaskStatusFromInterfaces identifies implementations of the FirmwareTaskVerifier interface and passes the found implementations to the firmwareTaskStatus() wrapper. -func FirmwareTaskStatusFromInterfaces(ctx context.Context, kind bconsts.FirmwareInstallStep, component, taskID, installVersion string, generic []interface{}) (state constants.TaskState, status string, metadata Metadata, err error) { +func FirmwareTaskStatusFromInterfaces(ctx context.Context, kind constants.FirmwareInstallStep, component, taskID, installVersion string, generic []interface{}) (state constants.TaskState, status string, metadata Metadata, err error) { metadata = newMetadata() implementations := make([]firmwareTaskVerifierProvider, 0) From f388f58b1a2635428c974f7a531343c76f1a078d Mon Sep 17 00:00:00 2001 From: Jake Schuurmans Date: Tue, 3 Dec 2024 13:03:24 -0500 Subject: [PATCH 5/5] fix x11 and x12 bug --- providers/supermicro/supermicro.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/supermicro/supermicro.go b/providers/supermicro/supermicro.go index 01602617..f851bdff 100644 --- a/providers/supermicro/supermicro.go +++ b/providers/supermicro/supermicro.go @@ -182,7 +182,8 @@ func (c *Client) Open(ctx context.Context) (err error) { return err } - if !bytes.Contains(body, []byte(`url_redirect.cgi?url_name=topmenu`)) { + if !bytes.Contains(body, []byte(`url_redirect.cgi?url_name=mainmenu`)) && + !bytes.Contains(body, []byte(`url_redirect.cgi?url_name=topmenu`)) { return closeWithError(ctx, errors.Wrap(bmclibErrs.ErrLoginFailed, "unexpected response contents")) } @@ -568,7 +569,6 @@ func (c *serviceClient) query(ctx context.Context, endpoint, method string, payl if cookie.Name == "SID" && cookie.Value != "" { req.AddCookie(cookie) } - } var reqDump []byte