Skip to content

Commit

Permalink
migratedb: add sqlite possibility
Browse files Browse the repository at this point in the history
  • Loading branch information
@ziggie1984
ziggie1984 committed Nov 22, 2024
1 parent 7c12213 commit db62172
Showing 1 changed file with 149 additions and 33 deletions.
182 changes: 149 additions & 33 deletions cmd_migrate_db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,15 @@ import (
"strconv"
"time"

"github.com/btcsuite/btcd/btcutil"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/jessevdk/go-flags"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/kvdb"
"github.com/lightningnetwork/lnd/kvdb/etcd"
"github.com/lightningnetwork/lnd/kvdb/postgres"
"github.com/lightningnetwork/lnd/kvdb/sqlbase"
"github.com/lightningnetwork/lnd/kvdb/sqlite"
"github.com/lightningnetwork/lnd/lncfg"
"github.com/lightningnetwork/lnd/signal"
clientv3 "go.etcd.io/etcd/client/v3"
Expand All @@ -27,6 +30,9 @@ var (

// etcdTimeout is the time we allow a single etcd transaction to take.
etcdTimeout = time.Second * 5

// defaultDataDir is the default data directory for lnd.
defaultDataDir = btcutil.AppDataDir("lnd", false)
)

const (
Expand All @@ -40,28 +46,53 @@ type Bolt struct {
DBTimeout time.Duration `long:"dbtimeout" description:"Specify the timeout value used when opening the database."`
DataDir string `long:"data-dir" description:"Lnd data dir where bolt dbs are located."`
TowerDir string `long:"tower-dir" description:"Lnd watchtower dir where bolt dbs for the watchtower server are located."`
Network string `long:"network" description:"Network within data dir where bolt dbs are located."`
}

type Sqlite struct {
DataDir string `long:"data-dir" description:"Lnd data dir where sqlite dbs are located."`
TowerDir string `long:"tower-dir" description:"Lnd watchtower dir where sqlite dbs for the watchtower server are located."`
Config *sqlite.Config `group:"sqlite-config" namespace:"sqlite-config" description:"Sqlite config."`
}

type DB struct {
Backend string `long:"backend" description:"The selected database backend."`
Etcd *etcd.Config `group:"etcd" namespace:"etcd" description:"Etcd settings."`
Bolt *Bolt `group:"bolt" namespace:"bolt" description:"Bolt settings."`
Postgres *postgres.Config `group:"postgres" namespace:"postgres" description:"Postgres settings."`
Sqlite *Sqlite `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."`
}

// Init should be called upon start to pre-initialize database for sql
// backends. If max connections are not set, the amount of connections will be
// unlimited however we only use one connection during the migration.
func (db *DB) Init() error {
// Start embedded etcd server if requested.
switch {
case db.Backend == lncfg.PostgresBackend:
sqlbase.Init(db.Postgres.MaxConnections)

case db.Backend == lncfg.SqliteBackend:
sqlbase.Init(db.Sqlite.Config.MaxConnections)
}

return nil
}

func (d *DB) isRemote() bool {
return d.Backend == lncfg.EtcdBackend ||
d.Backend == lncfg.PostgresBackend
// isBoltDB returns true if the db is a type bolt db.
func (db *DB) isRemote() bool {
return db.Backend == lncfg.EtcdBackend ||
db.Backend == lncfg.PostgresBackend ||
db.Backend == lncfg.SqliteBackend
}

func (d *DB) isEtcd() bool {
return d.Backend == lncfg.EtcdBackend
func (db *DB) isEtcd() bool {
return db.Backend == lncfg.EtcdBackend
}

type migrateDBCommand struct {
Source *DB `group:"source" namespace:"source" long:"source" short:"s" description:"The source database where the data is read from"`
Dest *DB `group:"dest" namespace:"dest" long:"dest" short:"d" description:"The destination database where the data is written to"`
Source *DB `group:"source" namespace:"source" long:"source" short:"s" description:"The source database where the data is read from"`
Dest *DB `group:"dest" namespace:"dest" long:"dest" short:"d" description:"The destination database where the data is written to"`
Network string `long:"network" short:"n" description:"Network of the db files to migrate (used to navigate into the right directory)"`
}

func newMigrateDBCommand() *migrateDBCommand {
Expand All @@ -71,9 +102,15 @@ func newMigrateDBCommand() *migrateDBCommand {
Etcd: &etcd.Config{},
Bolt: &Bolt{
DBTimeout: kvdb.DefaultDBTimeout,
Network: "mainnet",
TowerDir: defaultDataDir,
DataDir: defaultDataDir,
},
Postgres: &postgres.Config{},
Sqlite: &Sqlite{
Config: &sqlite.Config{},
TowerDir: defaultDataDir,
DataDir: defaultDataDir,
},
},
Dest: &DB{
Backend: lncfg.EtcdBackend,
Expand All @@ -82,10 +119,17 @@ func newMigrateDBCommand() *migrateDBCommand {
},
Bolt: &Bolt{
DBTimeout: kvdb.DefaultDBTimeout,
Network: "mainnet",
TowerDir: defaultDataDir,
DataDir: defaultDataDir,
},
Postgres: &postgres.Config{},
Sqlite: &Sqlite{
Config: &sqlite.Config{},
TowerDir: defaultDataDir,
DataDir: defaultDataDir,
},
},
Network: "regtest",
}
}

Expand Down Expand Up @@ -141,20 +185,21 @@ func (x *migrateDBCommand) Execute(_ []string) error {
for _, prefix := range prefixes {
log("Migrating DB with prefix %s", prefix)

srcDb, err := openDb(x.Source, prefix)
if err != nil {
if err == walletdb.ErrDbDoesNotExist &&
x.Source.Backend == lncfg.BoltBackend {
srcDb, err := openDb(x.Source, prefix, x.Network)

log("Skipping DB with prefix %s because "+
"source does not exist", prefix)
continue
}
return err
// This is only for now done for bolt dbs because other backends
// do not special case this error for now. For example if
// we do not run the watcher tower functionaliy we might not
// have a towerclient.db so we skip this error.
if err == walletdb.ErrDbDoesNotExist &&
x.Source.Backend == lncfg.BoltBackend {

log("Skipping DB with prefix %s because "+
"source does not exist", prefix)
continue
}
log("Opened source DB")

destDb, err := openDb(x.Dest, prefix)
destDb, err := openDb(x.Dest, prefix, x.Network)
if err != nil {
return err
}
Expand Down Expand Up @@ -183,11 +228,15 @@ func (x *migrateDBCommand) Execute(_ []string) error {
// to the channel DB as that is the only DB that has migrations.
log("Checking DB version of source DB")
if prefix == lncfg.NSChannelDB {
if err := checkMigrationsApplied(srcDb); err != nil {
err := checkChannelDBMigrationsApplied(srcDb)
if err != nil {
return err
}
}

// TODO(ziggie): Also check other DBs for migrations like the
// wtclient.db as soon as LND 19 is tagged.

// Also make sure that the destination DB hasn't been marked as
// successfully having been the target of a migration. We only
// mark a destination DB as successfully migrated at the end of
Expand Down Expand Up @@ -242,6 +291,9 @@ func (x *migrateDBCommand) Execute(_ []string) error {
if err := addMarker(destDb, alreadyMigratedKey); err != nil {
return err
}

log("Migration of DB with prefix %s completed successfully",
prefix)
}

return nil
Expand Down Expand Up @@ -382,21 +434,30 @@ func (x *migrateDBCommand) migrateEtcd(srcTx walletdb.ReadWriteTx,
return nil
}

func openDb(cfg *DB, prefix string) (walletdb.DB, error) {
func openDb(cfg *DB, prefix, network string) (walletdb.DB, error) {
backend := cfg.Backend

var args []interface{}
// Init the db connections for sql backends.
err := cfg.Init()
if err != nil {
return nil, err
}

graphDir := filepath.Join(cfg.Bolt.DataDir, "graph", cfg.Bolt.Network)
walletDir := filepath.Join(
cfg.Bolt.DataDir, "chain", "bitcoin", cfg.Bolt.Network,
)
towerServerDir := filepath.Join(
cfg.Bolt.TowerDir, "bitcoin", cfg.Bolt.Network,
)
// Settings to open a particular db backend.
var args []interface{}

switch backend {
case lncfg.BoltBackend:
// Directories where the db files are located.
graphDir := filepath.Join(cfg.Bolt.DataDir, "graph", network)
walletDir := filepath.Join(
cfg.Bolt.DataDir, "chain", "bitcoin", network,
)
towerServerDir := filepath.Join(
cfg.Bolt.TowerDir, "bitcoin", network,
)

// Path to the db file.
var path string
switch prefix {
case lncfg.NSChannelDB:
Expand Down Expand Up @@ -443,13 +504,68 @@ func openDb(cfg *DB, prefix string) (walletdb.DB, error) {
args = []interface{}{
context.Background(),
&postgres.Config{
Dsn: cfg.Postgres.Dsn,
Dsn: cfg.Postgres.Dsn,
Timeout: time.Minute,
MaxConnections: 10,
},
prefix,
}

log("Opening postgres backend at %s with prefix '%s'",
cfg.Postgres.Dsn, prefix)

case kvdb.SqliteBackendName:
// Directories where the db files are located.
graphDir := filepath.Join(cfg.Sqlite.DataDir, "graph", network)
walletDir := filepath.Join(
cfg.Sqlite.DataDir, "chain", "bitcoin", network,
)
towerServerDir := filepath.Join(
cfg.Sqlite.TowerDir, "bitcoin", network,
)

var dbName string
var path string
switch prefix {
case lncfg.NSChannelDB:
path = graphDir
dbName = lncfg.SqliteChannelDBName
case lncfg.NSMacaroonDB:
path = walletDir
dbName = lncfg.SqliteChainDBName

case lncfg.NSDecayedLogDB:
path = graphDir
dbName = lncfg.SqliteChannelDBName

case lncfg.NSTowerClientDB:
path = graphDir
dbName = lncfg.SqliteChannelDBName

case lncfg.NSTowerServerDB:
path = towerServerDir
dbName = lncfg.SqliteChannelDBName

case lncfg.NSWalletDB:
path = walletDir
dbName = lncfg.SqliteChainDBName

case lncfg.NSNeutrinoDB:
dbName = lncfg.SqliteNeutrinoDBName
}

args = []interface{}{
context.Background(),
&sqlite.Config{
Timeout: time.Minute,
},
path,
dbName,
prefix,
}

log("Opening sqlite backend with prefix '%s'", prefix)

default:
return nil, fmt.Errorf("unknown backend: %v", backend)
}
Expand Down Expand Up @@ -548,7 +664,7 @@ func addMarker(db walletdb.DB, markerKey []byte) error {
return rwtx.Commit()
}

func checkMigrationsApplied(db walletdb.DB) error {
func checkChannelDBMigrationsApplied(db walletdb.DB) error {
var meta channeldb.Meta
err := kvdb.View(db, func(tx kvdb.RTx) error {
return channeldb.FetchMeta(&meta, tx)
Expand Down

0 comments on commit db62172

Please sign in to comment.