diff --git a/Makefile b/Makefile index ecabab5..dfcf3c7 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,11 @@ XARGS := xargs -L 1 VERSION_TAG = $(shell git describe --tags) +DEV_TAGS = kvdb_etcd kvdb_postgres kvdb_sqlite +RELEASE_TAGS = $(DEV_TAGS) + BUILD_SYSTEM = darwin-amd64 \ +darwin-arm64 \ linux-386 \ linux-amd64 \ linux-armv6 \ @@ -49,7 +53,7 @@ endif make_ldflags = $(2) -X $(PKG).Commit=$(COMMIT) DEV_GCFLAGS := -gcflags "all=-N -l" -LDFLAGS := -ldflags "$(call make_ldflags, ${tags}, -s -w)" +LDFLAGS := -ldflags "$(call make_ldflags, $(DEV_TAGS), -s -w)" DEV_LDFLAGS := -ldflags "$(call make_ldflags, $(DEV_TAGS))" # For the release, we want to remove the symbol table and debug information (-s) @@ -83,7 +87,7 @@ build: install: @$(call print, "Installing lndinit.") - $(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG) + $(GOINSTALL) -tags="$(DEV_TAGS)" $(LDFLAGS) $(PKG) release-install: @$(call print, "Installing release lndinit.") @@ -105,7 +109,7 @@ scratch: build unit: @$(call print, "Running unit tests.") - $(GOTEST) ./... + $(GOTEST) -tags="$(DEV_TAGS)" ./... fmt: $(GOIMPORTS_BIN) @$(call print, "Fixing imports.") @@ -115,7 +119,7 @@ fmt: $(GOIMPORTS_BIN) lint: docker-tools @$(call print, "Linting source.") - $(DOCKER_TOOLS) golangci-lint run -v $(LINT_WORKERS) + $(DOCKER_TOOLS) golangci-lint run -v --build-tags="$(DEV_TAGS)"$(LINT_WORKERS) vendor: @$(call print, "Re-creating vendor directory.") diff --git a/README.md b/README.md index bfaec4f..4cdc2b4 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ initialization, including seed and password generation. - [`store-configmap`](#store-configmap) - [`init-wallet`](#init-wallet) - [`wait-ready`](#wait-ready) + - [`migrate-db`](#migrate-db) - [Example usage](#example-usage) - [Basic setup](#example-use-case-1-basic-setup) - [Kubernetes](#example-use-case-2-kubernetes) @@ -64,6 +65,11 @@ No `lnd` needed, but seed will be in `lnd`-specific [`aezeed` format](https://gi `wait-ready` waits for `lnd` to be ready by connecting to `lnd`'s status RPC - Needs `lnd` to run, eventually +### migrate-db +`migrate-db` migrates the content of one `lnd` database to another, for example +from `bbolt` to Postgres. See [data migration guide](docs/data-migration.md) for +more information. + --- ## Example Usage diff --git a/cmd_migrate_db.go b/cmd_migrate_db.go new file mode 100644 index 0000000..e2119f2 --- /dev/null +++ b/cmd_migrate_db.go @@ -0,0 +1,711 @@ +package main + +import ( + "context" + "encoding/hex" + "fmt" + "path/filepath" + "strconv" + "time" + + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcwallet/walletdb" + "github.com/jessevdk/go-flags" + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/kvdb/etcd" + "github.com/lightningnetwork/lnd/kvdb/postgres" + "github.com/lightningnetwork/lnd/kvdb/sqlbase" + "github.com/lightningnetwork/lnd/kvdb/sqlite" + "github.com/lightningnetwork/lnd/lncfg" + "github.com/lightningnetwork/lnd/signal" + clientv3 "go.etcd.io/etcd/client/v3" +) + +var ( + // alreadyMigratedKey is the key under which we add a tag in the target/ + // destination DB after we've successfully and completely migrated it + // from a source DB. + alreadyMigratedKey = []byte("data-migration-already-migrated") + + // etcdTimeout is the time we allow a single etcd transaction to take. + etcdTimeout = time.Second * 5 + + // defaultDataDir is the default data directory for lnd. + defaultDataDir = filepath.Join(btcutil.AppDataDir("lnd", false), "data") +) + +const ( + // EtcdMigrationMaxCallSize is the maximum size in bytes we allow a TX + // message for etcd to be. This must be large enough to accommodate the + // largest single value we ever expect to be in one of our databases. + EtcdMigrationMaxCallSize = 100 * 1024 * 1024 +) + +type Bolt struct { + DBTimeout time.Duration `long:"dbtimeout" description:"Specify the timeout value used when opening the database."` + DataDir string `long:"data-dir" description:"Lnd data dir where bolt dbs are located."` + TowerDir string `long:"tower-dir" description:"Lnd watchtower dir where bolt dbs for the watchtower server are located."` +} + +type Sqlite struct { + DataDir string `long:"data-dir" description:"Lnd data dir where sqlite dbs are located."` + TowerDir string `long:"tower-dir" description:"Lnd watchtower dir where sqlite dbs for the watchtower server are located."` + Config *sqlite.Config `group:"sqlite-config" namespace:"sqlite-config" description:"Sqlite config."` +} + +type DB struct { + Backend string `long:"backend" description:"The selected database backend."` + Etcd *etcd.Config `group:"etcd" namespace:"etcd" description:"Etcd settings."` + Bolt *Bolt `group:"bolt" namespace:"bolt" description:"Bolt settings."` + Postgres *postgres.Config `group:"postgres" namespace:"postgres" description:"Postgres settings."` + Sqlite *Sqlite `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."` +} + +// Init should be called upon start to pre-initialize database for sql +// backends. If max connections are not set, the amount of connections will be +// unlimited however we only use one connection during the migration. +func (db *DB) Init() error { + // Start embedded etcd server if requested. + switch { + case db.Backend == lncfg.PostgresBackend: + sqlbase.Init(db.Postgres.MaxConnections) + + case db.Backend == lncfg.SqliteBackend: + sqlbase.Init(db.Sqlite.Config.MaxConnections) + } + + return nil +} + +// isBoltDB returns true if the db is a type bolt db. +func (db *DB) isRemote() bool { + return db.Backend == lncfg.EtcdBackend || + db.Backend == lncfg.PostgresBackend || + db.Backend == lncfg.SqliteBackend +} + +func (db *DB) isEtcd() bool { + return db.Backend == lncfg.EtcdBackend +} + +type migrateDBCommand struct { + Source *DB `group:"source" namespace:"source" long:"source" short:"s" description:"The source database where the data is read from"` + Dest *DB `group:"dest" namespace:"dest" long:"dest" short:"d" description:"The destination database where the data is written to"` + Network string `long:"network" short:"n" description:"Network of the db files to migrate (used to navigate into the right directory)"` +} + +func newMigrateDBCommand() *migrateDBCommand { + return &migrateDBCommand{ + Source: &DB{ + Backend: lncfg.BoltBackend, + Etcd: &etcd.Config{}, + Bolt: &Bolt{ + DBTimeout: kvdb.DefaultDBTimeout, + TowerDir: defaultDataDir, + DataDir: defaultDataDir, + }, + Postgres: &postgres.Config{}, + Sqlite: &Sqlite{ + Config: &sqlite.Config{}, + TowerDir: defaultDataDir, + DataDir: defaultDataDir, + }, + }, + Dest: &DB{ + Backend: lncfg.EtcdBackend, + Etcd: &etcd.Config{ + MaxMsgSize: EtcdMigrationMaxCallSize, + }, + Bolt: &Bolt{ + DBTimeout: kvdb.DefaultDBTimeout, + TowerDir: defaultDataDir, + DataDir: defaultDataDir, + }, + Postgres: &postgres.Config{}, + Sqlite: &Sqlite{ + Config: &sqlite.Config{}, + TowerDir: defaultDataDir, + DataDir: defaultDataDir, + }, + }, + Network: "regtest", + } +} + +func (x *migrateDBCommand) Register(parser *flags.Parser) error { + _, err := parser.AddCommand( + "migrate-db", + "Migrate the complete database state of lnd to a new backend", + ` + Migrate the full database state of lnd from a source (for example the + set of bbolt database files such as channel.db and wallet.db) database + to a destination (for example a remote etcd or postgres) database. + + IMPORTANT: Please read the data migration guide located in the file + docs/data-migration.md of the main lnd repository before using this + command! + + NOTE: The migration can take a long time depending on the amount of data + that needs to be written! Because of the number of operations that need + to be completed, the migration cannot occur in a single database + transaction. Therefore the migration is not 100% atomic but happens + bucket by bucket. + As long as NEITHER the source nor destination database has been started/ + run with lnd, the migration can be repeated/resumed in case of an error + since the data will just be overwritten again in the destination. + + Once a database was successfully and completely migrated from the source + to the destination, the source will be marked with a 'tombstone' tag + while the destination will get an 'already migrated' tag. + A database with a tombstone cannot be started with lnd anymore to + prevent from an old state being used by accident. + To prevent overwriting a destination database by accident, the same + database/namespace pair cannot be used as the target of a data migration + twice, which is checked through the 'already migrated' tag. + `, + x, + ) + return err +} + +func (x *migrateDBCommand) Execute(_ []string) error { + // Since this will potentially run for a while, make sure we catch any + // interrupt signals. + _, err := signal.Intercept() + if err != nil { + return fmt.Errorf("error intercepting signals: %v", err) + } + + var prefixes = []string{ + lncfg.NSChannelDB, lncfg.NSMacaroonDB, lncfg.NSDecayedLogDB, + lncfg.NSTowerClientDB, lncfg.NSTowerServerDB, lncfg.NSWalletDB, + } + + for _, prefix := range prefixes { + log("Migrating DB with prefix %s", prefix) + + srcDb, err := openDb(x.Source, prefix, x.Network) + + // This is only for now done for bolt dbs because other backends + // do not special case this error for now. For example if + // we do not run the watcher tower functionaliy we might not + // have a towerclient.db so we skip this error. + if err == walletdb.ErrDbDoesNotExist && + x.Source.Backend == lncfg.BoltBackend { + + log("Skipping DB with prefix %s because "+ + "source does not exist", prefix) + continue + } + + destDb, err := openDb(x.Dest, prefix, x.Network) + if err != nil { + return err + } + log("Opened destination DB") + + // Check that the source database hasn't been marked with a + // tombstone yet. Once we set the tombstone we see the DB as not + // viable for migration anymore to avoid old state overwriting + // new state. We only set the tombstone at the end of a + // successful and complete migration. + log("Checking tombstone marker on source DB") + marker, err := checkMarkerPresent(srcDb, channeldb.TombstoneKey) + if err == nil { + log("Skipping DB with prefix %s because the source "+ + "DB was marked with a tombstone which "+ + "means it was already migrated successfully. "+ + "Tombstone reads: %s", prefix, marker) + continue + } + if err != channeldb.ErrMarkerNotPresent { + return err + } + + // Check that the source DB has had all its schema migrations + // applied before we migrate any of its data. This only applies + // to the channel DB as that is the only DB that has migrations. + log("Checking DB version of source DB") + if prefix == lncfg.NSChannelDB { + err := checkChannelDBMigrationsApplied(srcDb) + if err != nil { + return err + } + } + + // TODO(ziggie): Also check other DBs for migrations like the + // wtclient.db as soon as LND 19 is tagged. + + // Also make sure that the destination DB hasn't been marked as + // successfully having been the target of a migration. We only + // mark a destination DB as successfully migrated at the end of + // a successful and complete migration. + log("Checking if migration was already applied to target DB") + marker, err = checkMarkerPresent(destDb, alreadyMigratedKey) + if err == nil { + log("Skipping DB with prefix %s because the "+ + "destination DB was marked as already having "+ + "been the target of a successful migration. "+ + "Tag reads: %s", prefix, marker) + continue + } + if err != channeldb.ErrMarkerNotPresent { + return err + } + + // Using ReadWrite otherwise there is no access to the sequence + // number. + srcTx, err := srcDb.BeginReadWriteTx() + if err != nil { + return err + } + + if x.Dest.isEtcd() { + log("Starting the migration to the etcd backend") + err := x.migrateEtcd(srcTx, prefix) + if err != nil { + return err + } + } else { + log("Starting the migration to the target backend") + err := x.migrateKvdb(srcTx, destDb, prefix) + if err != nil { + return err + } + } + + // We're done now, so we can roll back the read transaction of + // the source DB. + if err := srcTx.Rollback(); err != nil { + return fmt.Errorf("error rolling back source tx: %v", + err) + } + + // If we get here, we've successfully migrated the DB and can + // now set the tombstone marker on the source database and the + // already migrated marker on the target database. + if err := addMarker(srcDb, channeldb.TombstoneKey); err != nil { + return err + } + if err := addMarker(destDb, alreadyMigratedKey); err != nil { + return err + } + + log("Migration of DB with prefix %s completed successfully", + prefix) + } + + return nil +} + +func (x *migrateDBCommand) migrateKvdb(srcTx walletdb.ReadWriteTx, + destDb walletdb.DB, prefix string) error { + + err := srcTx.ForEachBucket(func(key []byte) error { + log("Copying top-level bucket '%s'", loggableKeyName(key)) + + destTx, err := destDb.BeginReadWriteTx() + if err != nil { + return err + } + + destBucket, err := destTx.CreateTopLevelBucket(key) + if err != nil { + return fmt.Errorf("error creating top level bucket "+ + "'%s': %v", loggableKeyName(key), err) + } + + srcBucket := srcTx.ReadWriteBucket(key) + err = copyBucketKvdb(srcBucket, destBucket) + if err != nil { + return fmt.Errorf("error copying bucket '%s': %v", + loggableKeyName(key), err) + } + + log("Committing bucket '%s'", loggableKeyName(key)) + if err := destTx.Commit(); err != nil { + return fmt.Errorf("error committing bucket '%s': %v", + loggableKeyName(key), err) + } + + return nil + + }) + if err != nil { + return fmt.Errorf("error enumerating top level buckets: %v", + err) + } + + // Migrate wallet created marker. + if prefix == lncfg.NSWalletDB && x.Dest.isRemote() { + const ( + walletMetaBucket = "lnwallet" + walletReadyKey = "ready" + ) + + log("Creating 'wallet created' marker") + destTx, err := destDb.BeginReadWriteTx() + if err != nil { + return err + } + + metaBucket, err := destTx.CreateTopLevelBucket( + []byte(walletMetaBucket), + ) + if err != nil { + return err + } + + err = metaBucket.Put( + []byte(walletReadyKey), []byte(walletReadyKey), + ) + if err != nil { + return err + } + + log("Committing 'wallet created' marker") + if err := destTx.Commit(); err != nil { + return fmt.Errorf("error committing 'wallet created' "+ + "marker: %v", err) + } + } + + return nil +} + +func copyBucketKvdb(src walletdb.ReadWriteBucket, + dest walletdb.ReadWriteBucket) error { + + if err := dest.SetSequence(src.Sequence()); err != nil { + return fmt.Errorf("error copying sequence number") + } + + return src.ForEach(func(k, v []byte) error { + if v == nil { + srcBucket := src.NestedReadWriteBucket(k) + destBucket, err := dest.CreateBucket(k) + if err != nil { + return fmt.Errorf("error creating bucket "+ + "'%s': %v", loggableKeyName(k), err) + } + + if err := copyBucketKvdb(srcBucket, destBucket); err != nil { + return fmt.Errorf("error copying bucket "+ + "'%s': %v", loggableKeyName(k), err) + } + + return nil + } + + err := dest.Put(k, v) + if err != nil { + return fmt.Errorf("error copying key '%s': %v", + loggableKeyName(k), err) + } + + return nil + }) +} + +func (x *migrateDBCommand) migrateEtcd(srcTx walletdb.ReadWriteTx, + prefix string) error { + + ctx := context.Background() + cfg := x.Dest.Etcd.CloneWithSubNamespace(prefix) + destDb, ctx, cancel, err := etcd.NewEtcdClient(ctx, *cfg) + if err != nil { + return err + } + defer cancel() + + err = srcTx.ForEachBucket(func(key []byte) error { + log("Copying top-level bucket '%s'", loggableKeyName(key)) + + return migrateBucketEtcd( + ctx, destDb, []string{string(key)}, + srcTx.ReadWriteBucket(key), + ) + }) + if err != nil { + return err + } + + return nil +} + +func openDb(cfg *DB, prefix, network string) (walletdb.DB, error) { + backend := cfg.Backend + + // Init the db connections for sql backends. + err := cfg.Init() + if err != nil { + return nil, err + } + + // Settings to open a particular db backend. + var args []interface{} + + switch backend { + case lncfg.BoltBackend: + // Directories where the db files are located. + graphDir := filepath.Join(cfg.Bolt.DataDir, "graph", network) + walletDir := filepath.Join( + cfg.Bolt.DataDir, "chain", "bitcoin", network, + ) + towerServerDir := filepath.Join( + cfg.Bolt.TowerDir, "bitcoin", network, + ) + + // Path to the db file. + var path string + switch prefix { + case lncfg.NSChannelDB: + path = filepath.Join(graphDir, lncfg.ChannelDBName) + + case lncfg.NSMacaroonDB: + path = filepath.Join(walletDir, lncfg.MacaroonDBName) + + case lncfg.NSDecayedLogDB: + path = filepath.Join(graphDir, lncfg.DecayedLogDbName) + + case lncfg.NSTowerClientDB: + path = filepath.Join(graphDir, lncfg.TowerClientDBName) + + case lncfg.NSTowerServerDB: + path = filepath.Join( + towerServerDir, lncfg.TowerServerDBName, + ) + + case lncfg.NSWalletDB: + path = filepath.Join(walletDir, lncfg.WalletDBName) + } + + const ( + noFreelistSync = true + timeout = time.Minute + ) + + args = []interface{}{ + path, noFreelistSync, timeout, + } + backend = kvdb.BoltBackendName + log("Opening bbolt backend at %s for prefix '%s'", path, prefix) + + case kvdb.EtcdBackendName: + args = []interface{}{ + context.Background(), + cfg.Etcd.CloneWithSubNamespace(prefix), + } + log("Opening etcd backend at %s with namespace '%s'", + cfg.Etcd.Host, prefix) + + case kvdb.PostgresBackendName: + args = []interface{}{ + context.Background(), + &postgres.Config{ + Dsn: cfg.Postgres.Dsn, + Timeout: time.Minute, + MaxConnections: 10, + }, + prefix, + } + + log("Opening postgres backend at %s with prefix '%s'", + cfg.Postgres.Dsn, prefix) + + case kvdb.SqliteBackendName: + // Directories where the db files are located. + graphDir := filepath.Join(cfg.Sqlite.DataDir, "graph", network) + walletDir := filepath.Join( + cfg.Sqlite.DataDir, "chain", "bitcoin", network, + ) + towerServerDir := filepath.Join( + cfg.Sqlite.TowerDir, "bitcoin", network, + ) + + var dbName string + var path string + switch prefix { + case lncfg.NSChannelDB: + path = graphDir + dbName = lncfg.SqliteChannelDBName + case lncfg.NSMacaroonDB: + path = walletDir + dbName = lncfg.SqliteChainDBName + + case lncfg.NSDecayedLogDB: + path = graphDir + dbName = lncfg.SqliteChannelDBName + + case lncfg.NSTowerClientDB: + path = graphDir + dbName = lncfg.SqliteChannelDBName + + case lncfg.NSTowerServerDB: + path = towerServerDir + dbName = lncfg.SqliteChannelDBName + + case lncfg.NSWalletDB: + path = walletDir + dbName = lncfg.SqliteChainDBName + + case lncfg.NSNeutrinoDB: + dbName = lncfg.SqliteNeutrinoDBName + } + + args = []interface{}{ + context.Background(), + &sqlite.Config{ + Timeout: time.Minute, + }, + path, + dbName, + prefix, + } + + log("Opening sqlite backend with prefix '%s'", prefix) + + default: + return nil, fmt.Errorf("unknown backend: %v", backend) + } + + return kvdb.Open(backend, args...) +} + +func putKeyValueEtcd(ctx context.Context, cli *clientv3.Client, key, + value string) error { + + ctx, cancel := context.WithTimeout(ctx, etcdTimeout) + defer cancel() + + _, err := cli.Put(ctx, key, value) + return err +} + +func migrateBucketEtcd(ctx context.Context, cli *clientv3.Client, path []string, + bucket walletdb.ReadWriteBucket) error { + + err := putKeyValueEtcd( + ctx, cli, etcd.BucketKey(path...), etcd.BucketVal(path...), + ) + if err != nil { + return err + } + + var children []string + err = bucket.ForEach(func(k, v []byte) error { + key := string(k) + if v != nil { + err := putKeyValueEtcd( + ctx, cli, etcd.ValueKey(key, path...), + string(v), + ) + if err != nil { + return err + } + } else { + children = append(children, key) + } + + return nil + }) + if err != nil { + return err + } + + seq := bucket.Sequence() + if seq != 0 { + // Store the number as a string. + err := putKeyValueEtcd( + ctx, cli, etcd.SequenceKey(path...), + strconv.FormatUint(seq, 10), + ) + if err != nil { + return err + } + } + + for _, child := range children { + childPath := append(path, child) + err := migrateBucketEtcd( + ctx, cli, childPath, + bucket.NestedReadWriteBucket([]byte(child)), + ) + if err != nil { + return err + } + } + + return nil +} + +func checkMarkerPresent(db walletdb.DB, markerKey []byte) ([]byte, error) { + rtx, err := db.BeginReadTx() + if err != nil { + return nil, err + } + defer func() { _ = rtx.Rollback() }() + + return channeldb.CheckMarkerPresent(rtx, markerKey) +} + +func addMarker(db walletdb.DB, markerKey []byte) error { + rwtx, err := db.BeginReadWriteTx() + if err != nil { + return err + } + + markerValue := []byte(fmt.Sprintf("lndinit migrate-db %s", time.Now())) + if err := channeldb.AddMarker(rwtx, markerKey, markerValue); err != nil { + return err + } + + return rwtx.Commit() +} + +func checkChannelDBMigrationsApplied(db walletdb.DB) error { + var meta channeldb.Meta + err := kvdb.View(db, func(tx kvdb.RTx) error { + return channeldb.FetchMeta(&meta, tx) + }, func() { + meta = channeldb.Meta{} + }) + if err != nil { + return err + } + + if meta.DbVersionNumber != channeldb.LatestDBVersion() { + return fmt.Errorf("refusing to migrate source database with "+ + "version %d while latest known DB version is %d; "+ + "please upgrade the DB before using the data "+ + "migration tool", meta.DbVersionNumber, + channeldb.LatestDBVersion()) + } + + return nil +} + +// loggableKeyName returns a printable name of the given key. +func loggableKeyName(key []byte) string { + strKey := string(key) + if hasSpecialChars(strKey) { + return hex.EncodeToString(key) + } + + return strKey +} + +// hasSpecialChars returns true if any of the characters in the given string +// cannot be printed. +func hasSpecialChars(s string) bool { + for _, b := range s { + if !(b >= 'a' && b <= 'z') && !(b >= 'A' && b <= 'Z') && + !(b >= '0' && b <= '9') && b != '-' && b != '_' { + + return true + } + } + + return false +} diff --git a/docs/data-migration.md b/docs/data-migration.md new file mode 100644 index 0000000..2307aab --- /dev/null +++ b/docs/data-migration.md @@ -0,0 +1,105 @@ +# Data migration + +This document describes the process of migrating `lnd`'s database state from one +type of database backend (for example the `bbolt` based database files `*.db` +such as the `channel.db` or `wallet.db` files) to another (for example the new +remote database backends such as `etcd` or `postgres` introduced in +`lnd v0.14.0-beta`). + +## Prepare the destination database + +### Using etcd as the destination remote database + +When using `etcd` as the main remote database, some specific environment +variables need to be set to ensure smooth operations both during the data +migration and also later for running `lnd` in production. + +Make sure you are running **at least `etcd v3.5.0` or later** with the following +environment variables (or their command line flag counterparts) set: + +```shell +# Allow lnd to batch up to 16k operations into a single transaction with a max +# total TX size of 104MB. +ETCD_MAX_TXN_OPS=16384 +ETCD_MAX_REQUEST_BYTES=104857600 + +# Keep 10k revisions for raft consensus in clustered mode. +ETCD_AUTO_COMPACTION_RETENTION=10000 +ETCD_AUTO_COMPACTION_MODE=revision + +# Allow the total database size to be up to 15GB. Adjust this to your needs! +ETCD_QUOTA_BACKEND_BYTES=16106127360 +``` + +Make sure you set the `ETCD_QUOTA_BACKEND_BYTES` to something that is +sufficiently larger than your current size of the `channel.db` file! + +### Using postgres as the destination remote database + +Prepare a user and database as described in the [Postgres](postgres.md) +documentation. You'll need the Data Source Name (DSN) for both the data +migration and then the `lnd` configuration, so keep that string somewhere +(should be something with the format of `postgres://xx:yy@localhost:5432/zz`). + +No additional steps are required to prepare the Postgres database for the data +migration. The migration tool will create the database schema automatically, so +no DDL scripts need to be run in advance. + +## Prepare the source database + +Assuming we want to migrate the database state from the pre-0.16.0 individual +`bbolt` based `*.db` files to a remote database, we first need to make sure the +source files are in the correct state. + +The following steps should be performed *before* running the data migration: +1. Stop `lnd` +2. Upgrade the `lnd` binary to the latest version (e.g. `v0.16.0-beta` or later) +3. Make sure to add config options like `gc-canceled-invoices-on-startup=true` + and `db.bolt.auto-compact=true` to your `lnd.conf` to optimize the source + database size by removing canceled invoices and compacting it on startup. +4. Start `lnd` normally, using the flags mentioned above but not yet changing + any database backend related configuration options. Check the log that the + database schema was migrated successfully, for example: `Checking for + schema update: latest_version=XX, db_version=XX` +5. Remove any data from the source database that you can. The fewer entries are + in the source database, the quicker the migration will complete. For example + failed payments (or their failed HTLC attempts) can be removed with + `lncli deletepayments`. +6. Stop `lnd` again and make sure it isn't started again by accident during the + data migration (e.g. disable any `systemd` or other scripts that start/stop + `lnd`). + +NOTE: If you were using the experimental `etcd` cluster mode that was introduced +in `lnd v0.13.0` it is highly recommended starting a fresh node. While the data +can in theory be migrated from the partial/mixed `etcd` and `bbolt` based state +the migration tool does not support it. + +## Run the migration + +Depending on the destination database type, run the migration with a command +similar to one of the following examples: + +**Example: Migrate from `bbolt` to `etcd`:** + +```shell +⛰ lndinit -v migrate-db \ + --source.bolt.data-dir /home/myuser/.lnd/data \ + --source.bolt.tower-dir /home/myuser/.lnd/watchtower \ + --dest.backend etcd \ + --dest.etcd.host=my-etcd-cluster-address:2379 +``` +If you weren't running a watchtower server, you can remove the line with +`--source.bolt.tower-dir`. + +**Example: Migrate from `bbolt` to `postgres`:** + +```shell +⛰ lndinit -v migrate-db \ + --source.bolt.data-dir /home/myuser/.lnd/data \ + --source.bolt.tower-dir /home/myuser/.lnd/data/watchtower \ + --dest.backend postgres \ + --dest.postgres.dsn=postgres://postgres:postgres@localhost:5432/postgres +``` + +If you weren't running a watchtower server, you can remove the line with +`--source.bolt.tower-dir`. diff --git a/go.mod b/go.mod index 6fb2685..6b1abd6 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,16 @@ module github.com/lightninglabs/lndinit require ( github.com/btcsuite/btcd v0.24.2-beta.rc1.0.20240625142744-cc26860b4026 + github.com/btcsuite/btcd/btcutil v1.1.5 github.com/btcsuite/btcwallet v0.16.10-0.20240718224643-db3a4a2543bd + github.com/btcsuite/btcwallet/walletdb v1.4.2 github.com/jessevdk/go-flags v1.4.0 github.com/kkdai/bstream v1.0.0 github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display - github.com/lightningnetwork/lnd v0.18.3-beta.rc1 + github.com/lightningnetwork/lnd v0.18.3-beta + github.com/lightningnetwork/lnd/kvdb v1.4.10 github.com/stretchr/testify v1.9.0 + go.etcd.io/etcd/client/v3 v3.5.7 google.golang.org/grpc v1.59.0 k8s.io/api v0.18.3 k8s.io/apimachinery v0.18.3 @@ -23,14 +27,13 @@ require ( github.com/aead/siphash v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.3 // indirect - github.com/btcsuite/btcd/btcutil v1.1.5 // indirect github.com/btcsuite/btcd/btcutil/psbt v1.1.8 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect - github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect + github.com/btcsuite/btclog v0.0.0-20241003133417-09c4e92e319c // indirect + github.com/btcsuite/btclog/v2 v2.0.0-20241017175713-3428138b75c7 // indirect github.com/btcsuite/btcwallet/wallet/txauthor v1.3.4 // indirect github.com/btcsuite/btcwallet/wallet/txrules v1.2.1 // indirect github.com/btcsuite/btcwallet/wallet/txsizes v1.2.4 // indirect - github.com/btcsuite/btcwallet/walletdb v1.4.2 // indirect github.com/btcsuite/btcwallet/wtxmgr v1.5.3 // indirect github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect @@ -101,7 +104,6 @@ require ( github.com/lightningnetwork/lnd/clock v1.1.1 // indirect github.com/lightningnetwork/lnd/fn v1.2.0 // indirect github.com/lightningnetwork/lnd/healthcheck v1.2.5 // indirect - github.com/lightningnetwork/lnd/kvdb v1.4.10 // indirect github.com/lightningnetwork/lnd/queue v1.1.1 // indirect github.com/lightningnetwork/lnd/sqldb v1.0.3 // indirect github.com/lightningnetwork/lnd/ticker v1.1.1 // indirect @@ -144,7 +146,6 @@ require ( go.etcd.io/etcd/api/v3 v3.5.7 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect go.etcd.io/etcd/client/v2 v2.305.7 // indirect - go.etcd.io/etcd/client/v3 v3.5.7 // indirect go.etcd.io/etcd/pkg/v3 v3.5.7 // indirect go.etcd.io/etcd/raft/v3 v3.5.7 // indirect go.etcd.io/etcd/server/v3 v3.5.7 // indirect @@ -198,4 +199,6 @@ require ( // allows us to specify that as an option. replace google.golang.org/protobuf => github.com/lightninglabs/protobuf-go-hex-display v1.30.0-hex-display -go 1.21.10 +replace github.com/lightningnetwork/lnd/sqldb => github.com/lightningnetwork/lnd/sqldb v1.0.5 + +go 1.21.4 diff --git a/go.sum b/go.sum index cde7cf2..4ed5e51 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,11 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtyd github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btclog v0.0.0-20241003133417-09c4e92e319c h1:4HxD1lBUGUddhzgaNgrCPsFWd7cGYNpeFUgd9ZIgyM0= +github.com/btcsuite/btclog v0.0.0-20241003133417-09c4e92e319c/go.mod h1:w7xnGOhwT3lmrS4H3b/D1XAXxvh+tbhUm8xeHN2y3TQ= +github.com/btcsuite/btclog/v2 v2.0.0-20241017175713-3428138b75c7 h1:3Ct3zN3VCEKVm5nceWBBEKczc+jvTfVyOEG71ob2Yuc= +github.com/btcsuite/btclog/v2 v2.0.0-20241017175713-3428138b75c7/go.mod h1:XItGUfVOxotJL8kkuk2Hj3EVow5KCugXl3wWfQ6K0AE= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcwallet v0.16.10-0.20240718224643-db3a4a2543bd h1:QDb8foTCRoXrfoZVEzSYgSde16MJh4gCtCin8OCS0kI= github.com/btcsuite/btcwallet v0.16.10-0.20240718224643-db3a4a2543bd/go.mod h1:X2xDre+j1QphTRo54y2TikUzeSvreL1t1aMXrD8Kc5A= @@ -469,8 +472,8 @@ github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display h1:RZJ8H4ueU/aQ github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display/go.mod h1:2oKOBU042GKFHrdbgGiKax4xVrFiZu51lhacUZQ9MnE= github.com/lightningnetwork/lightning-onion v1.2.1-0.20240712235311-98bd56499dfb h1:yfM05S8DXKhuCBp5qSMZdtSwvJ+GFzl94KbXMNB1JDY= github.com/lightningnetwork/lightning-onion v1.2.1-0.20240712235311-98bd56499dfb/go.mod h1:c0kvRShutpj3l6B9WtTsNTBUtjSmjZXbJd9ZBRQOSKI= -github.com/lightningnetwork/lnd v0.18.3-beta.rc1 h1:ch6sQtld4NeSPDq359coDe/MW8gNTJjkuCmlb0xlrAw= -github.com/lightningnetwork/lnd v0.18.3-beta.rc1/go.mod h1:TbYgzDPPkyyWCk0Go2REoWh6zNR69BOq2eM+RKoCUvQ= +github.com/lightningnetwork/lnd v0.18.3-beta h1:I1Mcz79HGpVGPz0U2jSdxzzqzIi2cwUF0DXtzYJS7C8= +github.com/lightningnetwork/lnd v0.18.3-beta/go.mod h1:Xamph8AYM3iWyyn9w/tx+cLG6Tx1SSnSSPRFn71zuyQ= github.com/lightningnetwork/lnd/clock v1.1.1 h1:OfR3/zcJd2RhH0RU+zX/77c0ZiOnIMsDIBjgjWdZgA0= github.com/lightningnetwork/lnd/clock v1.1.1/go.mod h1:mGnAhPyjYZQJmebS7aevElXKTFDuO+uNFFfMXK1W8xQ= github.com/lightningnetwork/lnd/fn v1.2.0 h1:YTb2m8NN5ZiJAskHeBZAmR1AiPY8SXziIYPAX1VI/ZM= @@ -481,8 +484,8 @@ github.com/lightningnetwork/lnd/kvdb v1.4.10 h1:vK89IVv1oVH9ubQWU+EmoCQFeVRaC8kf github.com/lightningnetwork/lnd/kvdb v1.4.10/go.mod h1:J2diNABOoII9UrMnxXS5w7vZwP7CA1CStrl8MnIrb3A= github.com/lightningnetwork/lnd/queue v1.1.1 h1:99ovBlpM9B0FRCGYJo6RSFDlt8/vOkQQZznVb18iNMI= github.com/lightningnetwork/lnd/queue v1.1.1/go.mod h1:7A6nC1Qrm32FHuhx/mi1cieAiBZo5O6l8IBIoQxvkz4= -github.com/lightningnetwork/lnd/sqldb v1.0.3 h1:zLfAwOvM+6+3+hahYO9Q3h8pVV0TghAR7iJ5YMLCd3I= -github.com/lightningnetwork/lnd/sqldb v1.0.3/go.mod h1:4cQOkdymlZ1znnjuRNvMoatQGJkRneTj2CoPSPaQhWo= +github.com/lightningnetwork/lnd/sqldb v1.0.5 h1:ax5vBPf44tN/uD6C5+hBPBjOJ7cRMrUL+sVOdzmLVt4= +github.com/lightningnetwork/lnd/sqldb v1.0.5/go.mod h1:OG09zL/PHPaBJefp4HsPz2YLUJ+zIQHbpgCtLnOx8I4= github.com/lightningnetwork/lnd/ticker v1.1.1 h1:J/b6N2hibFtC7JLV77ULQp++QLtCwT6ijJlbdiZFbSM= github.com/lightningnetwork/lnd/ticker v1.1.1/go.mod h1:waPTRAAcwtu7Ji3+3k+u/xH5GHovTsCoSVpho0KDvdA= github.com/lightningnetwork/lnd/tlv v1.2.3 h1:If5ibokA/UoCBGuCKaY6Vn2SJU0l9uAbehCnhTZjEP8= diff --git a/main.go b/main.go index 97ca0f1..55e1353 100644 --- a/main.go +++ b/main.go @@ -112,8 +112,9 @@ func registerCommands(parser *flags.Parser) error { commands := []subCommand{ newGenPasswordCommand(), newGenSeedCommand(), - newLoadSecretCommand(), newInitWalletCommand(), + newLoadSecretCommand(), + newMigrateDBCommand(), newStoreSecretCommand(), newStoreConfigmapCommand(), newWaitReadyCommand(),