diff --git a/.gitignore b/.gitignore index 692c2fc..b082b79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store -bin \ No newline at end of file +bin +var \ No newline at end of file diff --git a/cmd/tidb/main.go b/cmd/tidb/main.go index 77d9a35..dd2cc48 100644 --- a/cmd/tidb/main.go +++ b/cmd/tidb/main.go @@ -20,7 +20,7 @@ import ( var ( requestCount = flag.Int("request-count", 500, "client test request count") runTime = flag.Duration("run-time", 10*time.Minute, "client test run time") - clientCase = flag.String("case", "bank", "client test case, like bank") + clientCase = flag.String("case", "bank", "client test case, like bank,multi_bank") historyFile = flag.String("history", "./history.log", "history file") nemesises = flag.String("nemesis", "", "nemesis, seperated by name, like random_kill,all_kill") verifyNames = flag.String("verifiers", "", "verifier names, seperate by comma, tidb_bank,tidb_bank_tso") @@ -44,6 +44,8 @@ func main() { switch *clientCase { case "bank": creator = tidb.BankClientCreator{} + case "multi_bank": + creator = tidb.MultiBankClientCreator{} default: log.Fatalf("invalid client test case %s", *clientCase) } diff --git a/cmd/verifier/verify/util.go b/cmd/verifier/verify/util.go index 477515e..7f14a6e 100644 --- a/cmd/verifier/verify/util.go +++ b/cmd/verifier/verify/util.go @@ -4,7 +4,6 @@ import ( "context" "log" "strings" - "sync" "github.com/siddontang/chaos/db/tidb" "github.com/siddontang/chaos/pkg/history" @@ -33,28 +32,23 @@ func Verify(ctx context.Context, historyFile string, verfier_names string) { childCtx, cancel := context.WithCancel(ctx) - var wg sync.WaitGroup - wg.Add(len(verifieres)) go func() { - wg.Wait() - cancel() - }() - - for _, verifier := range verifieres { - // Verify may take a long time, we should quit ASAP if receive signal. - go func(verifier history.Verifier) { - defer wg.Done() + for _, verifier := range verifieres { + log.Printf("begin to check with %s", verifier.Name()) ok, err := verifier.Verify(historyFile) if err != nil { log.Fatalf("verify history failed %v", err) } if !ok { - log.Fatalf("history %s is not linearizable", historyFile) + log.Fatalf("%s: history %s is not linearizable", verifier.Name(), historyFile) } else { - log.Printf("history %s is linearizable", historyFile) + log.Printf("%s: history %s is linearizable", verifier.Name(), historyFile) } - }(verifier) - } + } + + cancel() + }() + <-childCtx.Done() } diff --git a/db/tidb/bank.go b/db/tidb/bank.go index 9a9b415..ef83d1d 100644 --- a/db/tidb/bank.go +++ b/db/tidb/bank.go @@ -278,6 +278,11 @@ func (BankVerifier) Verify(historyFile string) (bool, error) { return history.VerifyHistory(historyFile, getBankModel(accountNum), bankParser{}) } +// Name returns the name of the verifier. +func (BankVerifier) Name() string { + return "bank_verifier" +} + // BankTsoVerifier verifies the bank history. // Unlike BankVerifier using porcupine, it uses a direct way because we know every timestamp of the transaction. // So we can order all transactions with timetamp and replay them. @@ -536,3 +541,8 @@ func (BankTsoVerifier) Verify(historyFile string) (bool, error) { return verifyTsoEvents(events), nil } + +// Name returns the name of the verifier. +func (BankTsoVerifier) Name() string { + return "bank_tso_verifier" +} diff --git a/db/tidb/multi_bank.go b/db/tidb/multi_bank.go new file mode 100644 index 0000000..9aee59d --- /dev/null +++ b/db/tidb/multi_bank.go @@ -0,0 +1,164 @@ +package tidb + +import ( + "context" + "database/sql" + "fmt" + "log" + "math/rand" + "time" + + "github.com/siddontang/chaos/pkg/core" +) + +type multiBankClient struct { + db *sql.DB + r *rand.Rand + accountNum int +} + +func (c *multiBankClient) SetUp(ctx context.Context, nodes []string, node string) error { + c.r = rand.New(rand.NewSource(time.Now().UnixNano())) + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(%s:4000)/test", node)) + if err != nil { + return err + } + c.db = db + + db.SetMaxIdleConns(1 + c.accountNum) + + // Do SetUp in the first node + if node != nodes[0] { + return nil + } + + log.Printf("begin to create table accounts on node %s", node) + + for i := 0; i < c.accountNum; i++ { + sql := fmt.Sprintf(`create table if not exists accounts_%d + (id int not null primary key, + balance bigint not null)`, i) + + if _, err = db.ExecContext(ctx, sql); err != nil { + return err + } + + sql = fmt.Sprintf("insert into accounts_%d values (?, ?)", i) + if _, err = db.ExecContext(ctx, sql, i, initBalance); err != nil { + return err + } + + } + + return nil +} + +func (c *multiBankClient) TearDown(ctx context.Context, nodes []string, node string) error { + return c.db.Close() +} + +func (c *multiBankClient) invokeRead(ctx context.Context, r bankRequest) bankResponse { + txn, err := c.db.Begin() + + if err != nil { + return bankResponse{Unknown: true} + } + defer txn.Rollback() + + var tso uint64 + if err = txn.QueryRow("select @@tidb_current_ts").Scan(&tso); err != nil { + return bankResponse{Unknown: true} + } + + balances := make([]int64, 0, c.accountNum) + for i := 0; i < c.accountNum; i++ { + var balance int64 + sql := fmt.Sprintf("select balance from accounts_%d", i) + if err = txn.QueryRowContext(ctx, sql).Scan(&balance); err != nil { + return bankResponse{Unknown: true} + } + balances = append(balances, balance) + } + + return bankResponse{Balances: balances, Tso: tso} +} + +func (c *multiBankClient) Invoke(ctx context.Context, node string, r interface{}) interface{} { + arg := r.(bankRequest) + if arg.Op == 0 { + return c.invokeRead(ctx, arg) + } + + txn, err := c.db.Begin() + + if err != nil { + return bankResponse{Ok: false} + } + defer txn.Rollback() + + var ( + fromBalance int64 + toBalance int64 + tso uint64 + ) + + if err = txn.QueryRow("select @@tidb_current_ts").Scan(&tso); err != nil { + return bankResponse{Ok: false} + } + + if err = txn.QueryRowContext(ctx, fmt.Sprintf("select balance from accounts_%d where id = ? for update", arg.From), arg.From).Scan(&fromBalance); err != nil { + return bankResponse{Ok: false} + } + + if err = txn.QueryRowContext(ctx, fmt.Sprintf("select balance from accounts_%d where id = ? for update", arg.To), arg.To).Scan(&toBalance); err != nil { + return bankResponse{Ok: false} + } + + if fromBalance < arg.Amount { + return bankResponse{Ok: false} + } + + if _, err = txn.ExecContext(ctx, fmt.Sprintf("update accounts_%d set balance = balance - ? where id = ?", arg.From), arg.Amount, arg.From); err != nil { + return bankResponse{Ok: false} + } + + if _, err = txn.ExecContext(ctx, fmt.Sprintf("update accounts_%d set balance = balance + ? where id = ?", arg.To), arg.Amount, arg.To); err != nil { + return bankResponse{Ok: false} + } + + if err = txn.Commit(); err != nil { + return bankResponse{Unknown: true, Tso: tso, FromBalance: fromBalance, ToBalance: toBalance} + } + + return bankResponse{Ok: true, Tso: tso, FromBalance: fromBalance, ToBalance: toBalance} +} + +func (c *multiBankClient) NextRequest() interface{} { + r := bankRequest{ + Op: c.r.Int() % 2, + } + if r.Op == 0 { + return r + } + + r.From = c.r.Intn(c.accountNum) + + r.To = c.r.Intn(c.accountNum) + if r.From == r.To { + r.To = (r.To + 1) % c.accountNum + } + + r.Amount = 5 + return r +} + +// MultiBankClientCreator creates a bank test client for tidb. +type MultiBankClientCreator struct { +} + +// Create creates a client. +func (MultiBankClientCreator) Create(node string) core.Client { + return &multiBankClient{ + accountNum: accountNum, + } +} diff --git a/pkg/history/history.go b/pkg/history/history.go index 6ca9881..e6b0d65 100644 --- a/pkg/history/history.go +++ b/pkg/history/history.go @@ -103,6 +103,7 @@ type RecordParser interface { // Verifier verifies the history. type Verifier interface { Verify(historyFile string) (bool, error) + Name() string } // VerifyHistory checks the history file with model. diff --git a/scripts/test_tidb.sh b/scripts/test_tidb.sh new file mode 100755 index 0000000..e5e7d04 --- /dev/null +++ b/scripts/test_tidb.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +cases=( bank multi_bank ) +nemeses=( random_kill random_drop ) +verifiers=( tidb_bank tidb_bank_tso ) + +mkdir -p var + +for i in "${cases[@]}" +do + for j in "${nemeses[@]}" + do + history_log=./var/history_"$i"_"$j".log + echo "run $i with nemeses $j" + ./bin/chaos-tidb --case $i --nemesis $j --history $history_log --request-count 200 + + for k in "${verifiers[@]}" + do + echo "use $k to check history" $history_log + ./bin/chaos-verifier --history $history_log --names $k + ret=$? + if [ $ret -ne 0 ]; then + exit $ret + fi + done + done +done \ No newline at end of file