diff --git a/cmd/tidb/main.go b/cmd/tidb/main.go index e7597d7..bdfb3ab 100644 --- a/cmd/tidb/main.go +++ b/cmd/tidb/main.go @@ -59,10 +59,10 @@ func main() { } switch name { - case "random_kill": - g = nemesis.NewRandomKillGenerator("tidb") - case "all_kill": - g = nemesis.NewAllKillGenerator("tidb") + case "random_kill", "all_kill", "minor_kill", "major_kill": + g = nemesis.NewKillGenerator("tidb", name) + case "random_drop", "all_drop", "minor_drop", "major_drop": + g = nemesis.NewDropGenerator(name) default: log.Fatalf("invalid nemesis generator") } diff --git a/pkg/nemesis/generator.go b/pkg/nemesis/generator.go index bc86c4c..4fd9d9b 100644 --- a/pkg/nemesis/generator.go +++ b/pkg/nemesis/generator.go @@ -7,57 +7,115 @@ import ( "github.com/siddontang/chaos/pkg/core" ) -type randomKillGenerator struct { - db string +type killGenerator struct { + db string + name string } -func (g randomKillGenerator) Generate(nodes []string) []*core.NemesisOperation { - index := rand.Intn(len(nodes)) +func (g killGenerator) Generate(nodes []string) []*core.NemesisOperation { + n := 1 + switch g.name { + case "minor_kill": + n = len(nodes)/2 - 1 + case "major_kill": + n = len(nodes)/2 + 1 + case "all_kill": + n = len(nodes) + default: + n = 1 + } + + return killNodes(g.db, nodes, n) +} +func (g killGenerator) Name() string { + return g.name +} + +func killNodes(db string, nodes []string, n int) []*core.NemesisOperation { ops := make([]*core.NemesisOperation, len(nodes)) - ops[index] = &core.NemesisOperation{ - Name: "kill", - InvokeArgs: []string{g.db}, - RecoverArgs: []string{g.db}, - RunTime: time.Second * time.Duration(rand.Intn(10)+1), + // randomly shuffle the indecies and get the first n nodes to be partitioned. + indices := shuffleIndices(len(nodes)) + + for i := 0; i < n; i++ { + ops[indices[i]] = &core.NemesisOperation{ + Name: "kill", + InvokeArgs: []string{db}, + RecoverArgs: []string{db}, + RunTime: time.Second * time.Duration(rand.Intn(10)+1), + } } return ops } -func (g randomKillGenerator) Name() string { - return "random_kill" +// NewKillGenerator creates a generator. +// Name is random_kill, minor_kill, major_kill, and all_kill. +func NewKillGenerator(db string, name string) core.NemesisGenerator { + return killGenerator{db: db, name: name} } -// NewRandomKillGenerator kills db in one node randomly. -func NewRandomKillGenerator(db string) core.NemesisGenerator { - return randomKillGenerator{db: db} +type dropGenerator struct { + name string } -type allKillGenerator struct { - db string +func (g dropGenerator) Generate(nodes []string) []*core.NemesisOperation { + n := 1 + switch g.name { + case "minor_drop": + n = len(nodes)/2 - 1 + case "major_drop": + n = len(nodes)/2 + 1 + case "all_drop": + n = len(nodes) + default: + n = 1 + } + return partitionNodes(nodes, n) } -func (g allKillGenerator) Generate(nodes []string) []*core.NemesisOperation { +func (g dropGenerator) Name() string { + return g.name +} + +func partitionNodes(nodes []string, n int) []*core.NemesisOperation { ops := make([]*core.NemesisOperation, len(nodes)) - for i := 0; i < len(ops); i++ { + // randomly shuffle the indecies and get the first n nodes to be partitioned. + indices := shuffleIndices(len(nodes)) + + partNodes := make([]string, n) + for i := 0; i < n; i++ { + partNodes[i] = nodes[indices[i]] + } + + for i := 0; i < len(nodes); i++ { ops[i] = &core.NemesisOperation{ - Name: "kill", - InvokeArgs: []string{g.db}, - RecoverArgs: []string{g.db}, - RunTime: time.Second * time.Duration(rand.Intn(10)+1), + Name: "drop", + InvokeArgs: partNodes, + RunTime: time.Second * time.Duration(rand.Intn(10)+1), } } + return ops } -func (g allKillGenerator) Name() string { - return "all_kill" +func shuffleIndices(n int) []int { + indices := make([]int, n) + for i := 0; i < n; i++ { + indices[i] = i + } + for i := len(indices) - 1; i > 0; i-- { + j := rand.Intn(i + 1) + indices[i], indices[j] = indices[j], indices[i] + } + + return indices } -// NewAllKillGenerator kills db in all nodes. -func NewAllKillGenerator(db string) core.NemesisGenerator { - return allKillGenerator{db: db} +// NewDropGenerator creates a generator. +// Name is random_drop, minor_drop, major_drop, and all_drop. +func NewDropGenerator(name string) core.NemesisGenerator { + return dropGenerator{name: name} } diff --git a/pkg/nemesis/nemesis.go b/pkg/nemesis/nemesis.go index 3c15203..59cfd2b 100644 --- a/pkg/nemesis/nemesis.go +++ b/pkg/nemesis/nemesis.go @@ -4,6 +4,7 @@ import ( "context" "github.com/siddontang/chaos/pkg/core" + "github.com/siddontang/chaos/pkg/util/net" ) type kill struct{} @@ -22,6 +23,33 @@ func (kill) Name() string { return "kill" } +type drop struct { + t net.IPTables +} + +func (n drop) Invoke(ctx context.Context, node string, args ...string) error { + for _, dropNode := range args { + if node == dropNode { + // Don't drop itself + continue + } + + if err := n.t.Drop(ctx, dropNode); err != nil { + return err + } + } + return nil +} + +func (n drop) Recover(ctx context.Context, node string, args ...string) error { + return n.t.Heal(ctx) +} + +func (drop) Name() string { + return "drop" +} + func init() { core.RegisterNemesis(kill{}) + core.RegisterNemesis(drop{}) } diff --git a/pkg/node/client.go b/pkg/node/client.go index 3af1f09..c612227 100644 --- a/pkg/node/client.go +++ b/pkg/node/client.go @@ -100,7 +100,10 @@ func (c *Client) IsDBRunning(name string) bool { func (c *Client) RunNemesis(op *core.NemesisOperation) error { v := url.Values{} suffix := fmt.Sprintf("/nemesis/%s/run", op.Name) - v.Set("dur", op.RunTime.String()) + if op.RunTime > 0 { + v.Set("dur", op.RunTime.String()) + } + if len(op.InvokeArgs) > 0 { v.Set("invoke_args", strings.Join(op.InvokeArgs, ",")) } diff --git a/pkg/node/nemesis.go b/pkg/node/nemesis.go index 51034c0..682dddf 100644 --- a/pkg/node/nemesis.go +++ b/pkg/node/nemesis.go @@ -3,6 +3,7 @@ package node import ( "fmt" "log" + "math/rand" "net/http" "strings" "time" @@ -49,7 +50,7 @@ func (h *nemesisHandler) Run(w http.ResponseWriter, r *http.Request) { recoverArgs := strings.Split(r.FormValue("recover_args"), ",") runTime, _ := time.ParseDuration(r.FormValue("dur")) if runTime == 0 { - runTime = 10 * time.Second + runTime = time.Second * time.Duration(rand.Intn(10)+1) } log.Printf("invoke nemesis %s with %v on node %s", nemesis.Name(), invokeArgs, node)