update kill and drop nemesis

cmd/tidb/main.go

@@ -59,10 +59,10 @@ func main() {
 		}
 
 		switch name {
-		case "random_kill":
-			g = nemesis.NewRandomKillGenerator("tidb")
-		case "all_kill":
-			g = nemesis.NewAllKillGenerator("tidb")
+		case "random_kill", "all_kill", "minor_kill", "major_kill":
+			g = nemesis.NewKillGenerator("tidb", name)
+		case "random_drop", "all_drop", "minor_drop", "major_drop":
+			g = nemesis.NewDropGenerator(name)
 		default:
 			log.Fatalf("invalid nemesis generator")
 		}

pkg/nemesis/generator.go

@@ -7,57 +7,115 @@ import (
 	"github.com/siddontang/chaos/pkg/core"
 )
 
-type randomKillGenerator struct {
-	db string
+type killGenerator struct {
+	db   string
+	name string
 }
 
-func (g randomKillGenerator) Generate(nodes []string) []*core.NemesisOperation {
-	index := rand.Intn(len(nodes))
+func (g killGenerator) Generate(nodes []string) []*core.NemesisOperation {
+	n := 1
+	switch g.name {
+	case "minor_kill":
+		n = len(nodes)/2 - 1
+	case "major_kill":
+		n = len(nodes)/2 + 1
+	case "all_kill":
+		n = len(nodes)
+	default:
+		n = 1
+	}
+
+	return killNodes(g.db, nodes, n)
+}
 
+func (g killGenerator) Name() string {
+	return g.name
+}
+
+func killNodes(db string, nodes []string, n int) []*core.NemesisOperation {
 	ops := make([]*core.NemesisOperation, len(nodes))
 
-	ops[index] = &core.NemesisOperation{
-		Name:        "kill",
-		InvokeArgs:  []string{g.db},
-		RecoverArgs: []string{g.db},
-		RunTime:     time.Second * time.Duration(rand.Intn(10)+1),
+	// randomly shuffle the indecies and get the first n nodes to be partitioned.
+	indices := shuffleIndices(len(nodes))
+
+	for i := 0; i < n; i++ {
+		ops[indices[i]] = &core.NemesisOperation{
+			Name:        "kill",
+			InvokeArgs:  []string{db},
+			RecoverArgs: []string{db},
+			RunTime:     time.Second * time.Duration(rand.Intn(10)+1),
+		}
 	}
 
 	return ops
 }
 
-func (g randomKillGenerator) Name() string {
-	return "random_kill"
+// NewKillGenerator creates a generator.
+// Name is random_kill, minor_kill, major_kill, and all_kill.
+func NewKillGenerator(db string, name string) core.NemesisGenerator {
+	return killGenerator{db: db, name: name}
 }
 
-// NewRandomKillGenerator kills db in one node randomly.
-func NewRandomKillGenerator(db string) core.NemesisGenerator {
-	return randomKillGenerator{db: db}
+type dropGenerator struct {
+	name string
 }
 
-type allKillGenerator struct {
-	db string
+func (g dropGenerator) Generate(nodes []string) []*core.NemesisOperation {
+	n := 1
+	switch g.name {
+	case "minor_drop":
+		n = len(nodes)/2 - 1
+	case "major_drop":
+		n = len(nodes)/2 + 1
+	case "all_drop":
+		n = len(nodes)
+	default:
+		n = 1
+	}
+	return partitionNodes(nodes, n)
 }
 
-func (g allKillGenerator) Generate(nodes []string) []*core.NemesisOperation {
+func (g dropGenerator) Name() string {
+	return g.name
+}
+
+func partitionNodes(nodes []string, n int) []*core.NemesisOperation {
 	ops := make([]*core.NemesisOperation, len(nodes))
 
-	for i := 0; i < len(ops); i++ {
+	// randomly shuffle the indecies and get the first n nodes to be partitioned.
+	indices := shuffleIndices(len(nodes))
+
+	partNodes := make([]string, n)
+	for i := 0; i < n; i++ {
+		partNodes[i] = nodes[indices[i]]
+	}
+
+	for i := 0; i < len(nodes); i++ {
 		ops[i] = &core.NemesisOperation{
-			Name:        "kill",
-			InvokeArgs:  []string{g.db},
-			RecoverArgs: []string{g.db},
-			RunTime:     time.Second * time.Duration(rand.Intn(10)+1),
+			Name:       "drop",
+			InvokeArgs: partNodes,
+			RunTime:    time.Second * time.Duration(rand.Intn(10)+1),
 		}
 	}
+
 	return ops
 }
 
-func (g allKillGenerator) Name() string {
-	return "all_kill"
+func shuffleIndices(n int) []int {
+	indices := make([]int, n)
+	for i := 0; i < n; i++ {
+		indices[i] = i
+	}
+	for i := len(indices) - 1; i > 0; i-- {
+		j := rand.Intn(i + 1)
+		indices[i], indices[j] = indices[j], indices[i]
+	}
+
+	return indices
 }
 
-// NewAllKillGenerator kills db in all nodes.
-func NewAllKillGenerator(db string) core.NemesisGenerator {
-	return allKillGenerator{db: db}
+// NewDropGenerator creates a generator.
+// Name is random_drop, minor_drop, major_drop, and all_drop.
+func NewDropGenerator(name string) core.NemesisGenerator {
+	return dropGenerator{name: name}
 }

pkg/nemesis/nemesis.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/siddontang/chaos/pkg/core"
+	"github.com/siddontang/chaos/pkg/util/net"
 )
 
 type kill struct{}
@@ -22,6 +23,33 @@ func (kill) Name() string {
 	return "kill"
 }
 
+type drop struct {
+	t net.IPTables
+}
+
+func (n drop) Invoke(ctx context.Context, node string, args ...string) error {
+	for _, dropNode := range args {
+		if node == dropNode {
+			// Don't drop itself
+			continue
+		}
+
+		if err := n.t.Drop(ctx, dropNode); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (n drop) Recover(ctx context.Context, node string, args ...string) error {
+	return n.t.Heal(ctx)
+}
+
+func (drop) Name() string {
+	return "drop"
+}
+
 func init() {
 	core.RegisterNemesis(kill{})
+	core.RegisterNemesis(drop{})
 }

pkg/node/client.go

@@ -100,7 +100,10 @@ func (c *Client) IsDBRunning(name string) bool {
 func (c *Client) RunNemesis(op *core.NemesisOperation) error {
 	v := url.Values{}
 	suffix := fmt.Sprintf("/nemesis/%s/run", op.Name)
-	v.Set("dur", op.RunTime.String())
+	if op.RunTime > 0 {
+		v.Set("dur", op.RunTime.String())
+	}
+
 	if len(op.InvokeArgs) > 0 {
 		v.Set("invoke_args", strings.Join(op.InvokeArgs, ","))
 	}

pkg/node/nemesis.go

@@ -3,6 +3,7 @@ package node
 import (
 	"fmt"
 	"log"
+	"math/rand"
 	"net/http"
 	"strings"
 	"time"
@@ -49,7 +50,7 @@ func (h *nemesisHandler) Run(w http.ResponseWriter, r *http.Request) {
 	recoverArgs := strings.Split(r.FormValue("recover_args"), ",")
 	runTime, _ := time.ParseDuration(r.FormValue("dur"))
 	if runTime == 0 {
-		runTime = 10 * time.Second
+		runTime = time.Second * time.Duration(rand.Intn(10)+1)
 	}
 
 	log.Printf("invoke nemesis %s with %v on node %s", nemesis.Name(), invokeArgs, node)