From 6d0273a25ba462c012ff14e2d6363b20dc2a3318 Mon Sep 17 00:00:00 2001 From: Nikita Savchenko Date: Fri, 3 Feb 2023 11:09:41 +0400 Subject: [PATCH] add local and remote mode for patroni attack --- cmd/attack/attack.go | 1 + cmd/attack/patroni.go | 12 +- pkg/core/patroni.go | 16 ++- pkg/server/chaosd/patroni.go | 126 +++++++++++------- .../utils/{status.go => http_requests.go} | 45 ++++++- 5 files changed, 140 insertions(+), 60 deletions(-) rename pkg/server/utils/{status.go => http_requests.go} (55%) diff --git a/cmd/attack/attack.go b/cmd/attack/attack.go index b7ed71a2..eca0be49 100644 --- a/cmd/attack/attack.go +++ b/cmd/attack/attack.go @@ -42,6 +42,7 @@ func NewAttackCommand() *cobra.Command { NewHTTPAttackCommand(&uid), NewVMAttackCommand(&uid), NewUserDefinedCommand(&uid), + NewPatroniAttackCommand(&uid), ) return cmd diff --git a/cmd/attack/patroni.go b/cmd/attack/patroni.go index 84260994..bb7849dd 100644 --- a/cmd/attack/patroni.go +++ b/cmd/attack/patroni.go @@ -48,6 +48,9 @@ func NewPatroniAttackCommand(uid *string) *cobra.Command { cmd.PersistentFlags().StringVarP(&options.User, "user", "u", "patroni", "patroni cluster user") cmd.PersistentFlags().StringVar(&options.Password, "password", "p", "patroni cluster password") + cmd.PersistentFlags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") + cmd.PersistentFlags().BoolVarP(&options.LocalMode, "local-mode", "l", false, "execute patronictl on host with chaosd. User with privileges required.") + cmd.PersistentFlags().BoolVarP(&options.RemoteMode, "remote-mode", "r", false, "execute patroni command by REST API") return cmd } @@ -55,15 +58,15 @@ func NewPatroniAttackCommand(uid *string) *cobra.Command { func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { cmd := &cobra.Command{ Use: "switchover", - Short: "exec switchover, default without another attack. Warning! Command is not recover!", + Short: "exec switchover command. Warning! Command is not recover!", Run: func(*cobra.Command, []string) { options.Action = core.SwitchoverAction utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() }, } - cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") cmd.Flags().StringVarP(&options.Candidate, "candidate", "c", "", "switchover candidate, default random unit for replicas") - cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprintln(time.Now().Add(time.Second*60).Format(time.RFC3339)), "scheduled switchover, default now()+1 minute") + cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprint(time.Now().Add(time.Second*60).Format(time.RFC3339)), `scheduled switchover, + default now()+1 minute by remote mode`) return cmd } @@ -71,14 +74,13 @@ func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *c func NewPatroniFailoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { cmd := &cobra.Command{ Use: "failover", - Short: "exec failover, default without another attack", + Short: "Exec failover command. Warning! Command is not recover!", Run: func(*cobra.Command, []string) { options.Action = core.FailoverAction utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() }, } - cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") cmd.Flags().StringVarP(&options.Candidate, "leader", "c", "", "failover new leader, default random unit for replicas") return cmd } diff --git a/pkg/core/patroni.go b/pkg/core/patroni.go index 5305a491..cffbb991 100644 --- a/pkg/core/patroni.go +++ b/pkg/core/patroni.go @@ -35,6 +35,8 @@ type PatroniCommand struct { User string `json:"user,omitempty"` Password string `json:"password,omitempty"` Scheduled_at string `json:"scheduled_at,omitempty"` + LocalMode bool `json:"local_mode,omitempty"` + RemoteMode bool `json:"remote_mode,omitempty"` RecoverCmd string `json:"recoverCmd,omitempty"` } @@ -46,12 +48,18 @@ func (p *PatroniCommand) Validate() error { return errors.New("address not provided") } - if len(p.User) == 0 { - return errors.New("patroni user not provided") + if !p.RemoteMode && !p.LocalMode { + return errors.New("local or remote mode required") } - if len(p.Password) == 0 { - return errors.New("patroni password not provided") + if p.RemoteMode { + if len(p.User) == 0 { + return errors.New("patroni user not provided") + } + + if len(p.Password) == 0 { + return errors.New("patroni password not provided") + } } return nil diff --git a/pkg/server/chaosd/patroni.go b/pkg/server/chaosd/patroni.go index 040f957f..e1cab7b1 100644 --- a/pkg/server/chaosd/patroni.go +++ b/pkg/server/chaosd/patroni.go @@ -14,12 +14,12 @@ package chaosd import ( - "bytes" "encoding/json" "fmt" "io" "math/rand" - "net/http" + "os/exec" + "strings" "github.com/chaos-mesh/chaosd/pkg/core" "github.com/chaos-mesh/chaosd/pkg/server/utils" @@ -34,17 +34,19 @@ var PatroniAttack AttackType = patroniAttack{} func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error { attack := options.(*core.PatroniCommand) - candidate := attack.Candidate + var responce []byte - leader := attack.Leader - - var scheduled_at string - - var url string + var address string values := make(map[string]string) - patroniInfo, err := utils.GetPatroniInfo(attack.Address) + if attack.RemoteMode { + address = attack.Address + } else if attack.LocalMode { + address = "localhost" + } + + patroniInfo, err := utils.GetPatroniInfo(address) if err != nil { err = errors.Errorf("failed to get patroni info for %v: %v", options.String(), err) return errors.WithStack(err) @@ -55,75 +57,109 @@ func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error { return errors.WithStack(err) } - if candidate == "" { - candidate = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))] + if attack.Candidate == "" { + values["candidate"] = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))] } - if leader == "" { - leader = patroniInfo.Master + if attack.Leader == "" { + values["leader"] = patroniInfo.Master } - switch options.String() { - case "switchover": + values["scheduled_at"] = attack.Scheduled_at - scheduled_at = attack.Scheduled_at + cmd := options.String() - values = map[string]string{"leader": leader, "scheduled_at": scheduled_at} + switch cmd { + case "switchover": - log.Info(fmt.Sprintf("Switchover will be done from %v to another available replica in %v", patroniInfo.Master, scheduled_at)) + log.Info(fmt.Sprintf("Switchover will be done from %v to %v in %v", values["leader"], values["candidate"], values["scheduled_at"])) case "failover": - values = map[string]string{"candidate": candidate} + log.Info(fmt.Sprintf("Failover will be done from %v to %v", values["leader"], values["candidate"])) - log.Info(fmt.Sprintf("Failover will be done from %v to %v", patroniInfo.Master, candidate)) + } + if attack.RemoteMode { + responce, err = execPatroniAttackByRemoteMode(attack, cmd, values) + if err != nil { + return err + } + } else if attack.LocalMode { + responce, err = execPatroniAttackByLocalMode(attack, cmd, values) + if err != nil { + return err + } } - patroniAddr := attack.Address + if attack.RemoteMode { + log.S().Infof("Execute %v successfully: %v", cmd, string(responce)) + } - cmd := options.String() + if attack.LocalMode { + log.S().Infof("Execute %v successfully", cmd) + fmt.Println(string(responce)) + } + + return nil +} + +func execPatroniAttackByRemoteMode(attack *core.PatroniCommand, cmd string, values map[string]string) ([]byte, error) { + patroniAddr := attack.Address data, err := json.Marshal(values) if err != nil { err = errors.Errorf("failed to marshal data: %v", values) - return errors.WithStack(err) + return nil, errors.WithStack(err) } - url = fmt.Sprintf("http://%v:8008/%v", patroniAddr, cmd) - - request, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + resp, err := utils.MakePostHTTPRequest(patroniAddr, 8008, cmd, data, attack.User, attack.Password) if err != nil { - err = errors.Errorf("failed to %v: %v", cmd, err) - return errors.WithStack(err) + return nil, errors.WithStack(err) } - request.Header.Set("Content-Type", "application/json") - request.SetBasicAuth(attack.User, attack.Password) - - client := &http.Client{} - resp, error := client.Do(request) - if error != nil { - err = errors.Errorf("failed to %v: %v", cmd, err) - return errors.WithStack(err) + if resp.StatusCode != 200 && resp.StatusCode != 202 { + //to simplify diagnostics + buf, err := io.ReadAll(resp.Body) + if err != nil { + err = errors.Errorf("failed to read from %s responce: status code %v, responce %v, error %v", cmd, resp.StatusCode, resp.Body, err) + return nil, err + } + err = errors.Errorf("failed to exec %v request: status code %v, responce %v", cmd, resp.StatusCode, buf) + return nil, errors.WithStack(err) } - defer resp.Body.Close() - buf, err := io.ReadAll(resp.Body) if err != nil { - err = errors.Errorf("failed to read %v responce: %v", cmd, err) - return errors.WithStack(err) + err = errors.Errorf("failed to read %v from %s responce: %v", resp.Body, cmd, err) + return nil, errors.WithStack(err) } - if resp.StatusCode != 200 && resp.StatusCode != 202 { - err = errors.Errorf("failed to %v: status code %v, responce %v", cmd, resp.StatusCode, string(buf)) - return errors.WithStack(err) + return buf, nil +} + +func execPatroniAttackByLocalMode(attack *core.PatroniCommand, cmd string, values map[string]string) ([]byte, error) { + var cmdTemplate string + + if cmd == "failover" { + cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --force", cmd, values["leader"], values["candidate"]) + } else if cmd == "switchover" { + cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --scheduled %v --force", cmd, values["leader"], values["candidate"], values["scheduled_at"]) + } + + execCmd := exec.Command("bash", "-c", cmdTemplate) + output, err := execCmd.CombinedOutput() + if err != nil { + log.S().Errorf(fmt.Sprintf("failed to %v: %v", cmdTemplate, string(output))) + return nil, err } - log.S().Infof("Execute %v successfully: %v", cmd, string(buf)) + if strings.Contains(string(output), "failed") { + err = errors.New(string(output)) + return nil, err + } - return nil + return output, nil } func (patroniAttack) Recover(exp core.Experiment, _ Environment) error { diff --git a/pkg/server/utils/status.go b/pkg/server/utils/http_requests.go similarity index 55% rename from pkg/server/utils/status.go rename to pkg/server/utils/http_requests.go index f49bd033..baa36884 100644 --- a/pkg/server/utils/status.go +++ b/pkg/server/utils/http_requests.go @@ -10,9 +10,11 @@ // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. + package utils import ( + "bytes" "fmt" "io" "net/http" @@ -23,9 +25,10 @@ import ( ) type PatroniInfo struct { - Master string - Replicas []string - Status []string + Master string + Replicas []string + SyncStandby []string + Status []string } func GetPatroniInfo(address string) (PatroniInfo, error) { @@ -50,17 +53,47 @@ func GetPatroniInfo(address string) (PatroniInfo, error) { members := gjson.Get(data, "members") for _, member := range members.Array() { - if member.Get("role").Str == "leader" { + switch member.Get("role").Str { + case "leader": patroniInfo.Master = member.Get("name").Str patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) - } else if member.Get("role").Str == "replica" || member.Get("role").Str == "sync_standby" { + case "replica": patroniInfo.Replicas = append(patroniInfo.Replicas, member.Get("name").Str) patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + case "sync_standby": + patroniInfo.SyncStandby = append(patroniInfo.SyncStandby, member.Get("name").Str) + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + } } - log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas, patroniInfo.Status)) + log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, sync_standy %s, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas, + patroniInfo.SyncStandby, patroniInfo.Status)) return patroniInfo, nil } + +func MakePostHTTPRequest(address string, port int64, path string, body []byte, user string, password string) (*http.Response, error) { + url := fmt.Sprintf("http://%v:%v/%v", address, port, path) + + request, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) + if err != nil { + err = errors.Errorf("failed to request %v: %v", url, err) + return nil, errors.WithStack(err) + } + + request.Header.Set("Content-Type", "application/json") + request.SetBasicAuth(user, password) + + client := &http.Client{} + resp, error := client.Do(request) + if error != nil { + err = errors.Errorf("failed to exec request %v: %v", url, err) + return nil, errors.WithStack(err) + } + + defer resp.Body.Close() + + return resp, nil +}