Consistent snapshot using retries instead of DBLog algorithm

It was hard to get the DBLog algorithm to work.

cmd/cloner/main.go

@@ -18,6 +18,7 @@ const TimestampFormat = `2006-01-02T15:04:05.000`
 
 var cli struct {
 	Clone     clone.Clone     `cmd:"" help:"Best effort copy of databases"`
+	NewClone  clone.NewClone  `cmd:"" help:"Best effort copy of databases"`
 	Checksum  clone.Checksum  `cmd:"" help:"Find differences between databases"`
 	Replicate clone.Replicate `cmd:"" help:"Replicate from one database to another and consistent clone"`
 	Ping      clone.Ping      `cmd:"" help:"Ping the databases to check the config is right"`

pkg/clone/clone.go

@@ -256,23 +256,3 @@ func (cmd *Clone) copyTableSchema(ctx context.Context, table *Table, source *sql
 	}
 	return nil
 }
-
-func removeElement[T comparable](slice []T, element T) []T {
-	return removeElementByIndex(slice, findIndex(slice, func(t T) bool {
-		return element == t
-	}))
-}
-
-func removeElementByIndex[T any](slice []T, index int) []T {
-	return append(slice[:index], slice[index+1:]...)
-}
-
-func findIndex[T any](slice []T, matchFunc func(T) bool) int {
-	for index, element := range slice {
-		if matchFunc(element) {
-			return index
-		}
-	}
-
-	return -1 // not found
-}

pkg/clone/clone_test.go

@@ -2,144 +2,13 @@ package clone
 
 import (
 	"context"
-	"database/sql"
 	"testing"
 	"time"
 
-	"github.com/mightyguava/autotx"
-	"vitess.io/vitess/go/vt/proto/topodata"
-
 	"github.com/alecthomas/kong"
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
-	"vitess.io/vitess/go/vt/key"
 )
 
-func insertBunchaData(ctx context.Context, config DBConfig, rowCount int) error {
-	db, err := config.DB()
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	defer db.Close()
-
-	err = autotx.Transact(ctx, db, func(tx *sql.Tx) error {
-		for i := 0; i < rowCount; i++ {
-			result, err := tx.ExecContext(ctx, `
-				INSERT INTO customers (name) VALUES (CONCAT('Customer ', LEFT(MD5(RAND()), 8)))
-			`)
-			if err != nil {
-				return errors.WithStack(err)
-			}
-			customerID, err := result.LastInsertId()
-			if err != nil {
-				return errors.WithStack(err)
-			}
-
-			// Insert a new row
-			_, err = tx.ExecContext(ctx, `
-				INSERT INTO transactions (customer_id, amount_cents, description) 
-				VALUES (?, RAND()*9999+1, CONCAT('Description ', LEFT(MD5(RAND()), 8)))
-			`, customerID)
-			if err != nil {
-				return errors.WithStack(err)
-			}
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	return nil
-}
-
-func clearTables(ctx context.Context, config DBConfig) error {
-	db, err := config.DB()
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	defer db.Close()
-
-	err = autotx.Transact(ctx, db, func(tx *sql.Tx) error {
-		_, err = tx.ExecContext(ctx, `DELETE FROM customers`)
-		if err != nil {
-			return errors.WithStack(err)
-		}
-		_, err = tx.ExecContext(ctx, `DELETE FROM transactions`)
-		if err != nil {
-			return errors.WithStack(err)
-		}
-		return nil
-	})
-
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	return nil
-}
-
-func countRows(target DBConfig, tableName string) (int, error) {
-	db, err := target.DB()
-	if err != nil {
-		return 0, errors.WithStack(err)
-	}
-	defer db.Close()
-	ctx := context.Background()
-	conn, err := db.Conn(ctx)
-	if err != nil {
-		return 0, errors.WithStack(err)
-	}
-	row := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tableName)
-	var rowCount int
-	err = row.Scan(&rowCount)
-	return rowCount, err
-}
-
-func countRowsShardFilter(target DBConfig, tableName string, shard string) (int, error) {
-	db, err := target.DB()
-	if err != nil {
-		return 0, errors.WithStack(err)
-	}
-	defer db.Close()
-	ctx := context.Background()
-	conn, err := db.Conn(ctx)
-	if err != nil {
-		return 0, errors.WithStack(err)
-	}
-	spec, err := key.ParseShardingSpec(shard)
-	if err != nil {
-		return 0, errors.WithStack(err)
-	}
-	rows, err := conn.QueryContext(ctx, "SELECT id FROM "+tableName)
-	if err != nil {
-		return 0, errors.WithStack(err)
-	}
-	defer rows.Close()
-	var rowCount int
-	for rows.Next() {
-		var id int64
-		err = rows.Scan(&id)
-		if err != nil {
-			return 0, errors.WithStack(err)
-		}
-		if inShard(uint64(id), spec) {
-			rowCount++
-		}
-	}
-	return rowCount, err
-}
-
-func inShard(id uint64, shard []*topodata.KeyRange) bool {
-	destination := key.DestinationKeyspaceID(vhash(id))
-	for _, keyRange := range shard {
-		if key.KeyRangeContains(keyRange, destination) {
-			return true
-		}
-	}
-	return false
-}
-
 func TestOneShardCloneWithTargetData(t *testing.T) {
 	_, _, err := startAll()
 	assert.NoError(t, err)