-
Notifications
You must be signed in to change notification settings - Fork 39
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
cb315c1
commit 1cc3481
Showing
18 changed files
with
480 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package database | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "errors" | ||
| "github.com/ten-protocol/go-ten/go/common/errutil" | ||
| ) | ||
|
|
||
| const ( | ||
| cfgInsert = "insert into config values (?,?)" | ||
| cfgSelect = "select val from config where ky=?" | ||
| ) | ||
|
|
||
| func WriteConfigToTx(dbtx *sql.Tx, key string, value any) (sql.Result, error) { | ||
| return dbtx.Exec(cfgInsert, key, value) | ||
| } | ||
|
|
||
| func WriteConfig(db *sql.DB, key string, value []byte) (sql.Result, error) { | ||
| return db.Exec(cfgInsert, key, value) | ||
| } | ||
|
|
||
| func FetchConfig(db *sql.DB, key string) ([]byte, error) { | ||
| return readSingleRow(db, cfgSelect, key) | ||
| } | ||
|
|
||
| func readSingleRow(db *sql.DB, query string, v any) ([]byte, error) { | ||
| var res []byte | ||
|
|
||
| err := db.QueryRow(query, v).Scan(&res) | ||
| if err != nil { | ||
| if errors.Is(err, sql.ErrNoRows) { | ||
| // make sure the error is converted to obscuro-wide not found error | ||
| return nil, errutil.ErrNotFound | ||
| } | ||
| return nil, err | ||
| } | ||
| return res, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| package sqlite | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "embed" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/ten-protocol/go-ten/go/common/log" | ||
|
|
||
| "github.com/ten-protocol/go-ten/go/enclave/storage/init/migration" | ||
|
|
||
| "github.com/ten-protocol/go-ten/go/enclave/storage/enclavedb" | ||
|
|
||
| gethlog "github.com/ethereum/go-ethereum/log" | ||
| "github.com/ten-protocol/go-ten/go/common" | ||
|
|
||
| _ "github.com/mattn/go-sqlite3" // this imports the sqlite driver to make the sql.Open() connection work | ||
| ) | ||
|
|
||
| const ( | ||
| tempDirName = "ten-persistence" | ||
| ) | ||
|
|
||
| //go:embed *.sql | ||
| var sqlFiles embed.FS | ||
|
|
||
| // CreateTemporarySQLiteEnclaveDB if dbPath is empty will use a random throwaway temp file, | ||
| // otherwise dbPath is a filepath for the sqldb file, allows for tests that care about persistence between restarts | ||
| func CreateTemporarySQLiteEnclaveDB(dbPath string, dbOptions string, logger gethlog.Logger, initFile string) (enclavedb.EnclaveDB, error) { | ||
| initialsed := false | ||
|
|
||
| if dbPath == "" { | ||
| tempPath, err := CreateTempDBFile() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create temp sqlite DB file - %w", err) | ||
| } | ||
| dbPath = tempPath | ||
| } | ||
|
|
||
| inMem := strings.Contains(dbOptions, "mode=memory") | ||
| description := "in memory" | ||
| if !inMem { | ||
| _, err := os.Stat(dbPath) | ||
| if err == nil { | ||
| description = "existing" | ||
| initialsed = true | ||
| } else { | ||
| description = "new" | ||
| } | ||
| } | ||
|
|
||
| db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?%s", dbPath, dbOptions)) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("couldn't open sqlite db - %w", err) | ||
| } | ||
|
|
||
| // Sqlite fails with table locks when there are multiple connections | ||
| db.SetMaxOpenConns(1) | ||
|
|
||
| if !initialsed { | ||
| err = initialiseDB(db, initFile) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| // perform db migration | ||
| err = migration.DBMigration(db, sqlFiles, logger.New(log.CmpKey, "DB_MIGRATION")) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| logger.Info(fmt.Sprintf("Opened %s sqlite db file at %s", description, dbPath)) | ||
|
|
||
| return enclavedb.NewEnclaveDB(db, logger) | ||
| } | ||
|
|
||
| // CreateTemporarySQLiteHostDB if dbPath is empty will use a random throwaway temp file, | ||
| // otherwise dbPath is a filepath for the sqldb file, allows for tests that care about persistence between restarts | ||
| func CreateTemporarySQLiteHostDB(dbPath string, dbOptions string, logger gethlog.Logger, initFile string) (*sql.DB, error) { | ||
| initialsed := false | ||
|
|
||
| if dbPath == "" { | ||
| tempPath, err := CreateTempDBFile() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create temp sqlite DB file - %w", err) | ||
| } | ||
| dbPath = tempPath | ||
| } | ||
|
|
||
| inMem := strings.Contains(dbOptions, "mode=memory") | ||
| description := "in memory" | ||
| if !inMem { | ||
| _, err := os.Stat(dbPath) | ||
| if err == nil { | ||
| description = "existing" | ||
| initialsed = true | ||
| } else { | ||
| description = "new" | ||
| } | ||
| } | ||
|
|
||
| db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?%s", dbPath, dbOptions)) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("couldn't open sqlite db - %w", err) | ||
| } | ||
|
|
||
| // Sqlite fails with table locks when there are multiple connections | ||
| db.SetMaxOpenConns(1) | ||
|
|
||
| if !initialsed { | ||
| err = initialiseDB(db, initFile) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| // perform db migration | ||
| err = migration.DBMigration(db, sqlFiles, logger.New(log.CmpKey, "DB_MIGRATION")) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| logger.Info(fmt.Sprintf("Opened %s sqlite db file at %s", description, dbPath)) | ||
|
|
||
| return db, nil | ||
| } | ||
|
|
||
| func initialiseDB(db *sql.DB, initFile string) error { | ||
| sqlInitFile, err := sqlFiles.ReadFile(initFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, err = db.Exec(string(sqlInitFile)) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to initialise sqlite %s - %w", sqlInitFile, err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func CreateTempDBFile() (string, error) { | ||
| tempDir := filepath.Join("/tmp", tempDirName, common.RandomStr(5)) | ||
| err := os.MkdirAll(tempDir, os.ModePerm) | ||
| if err != nil { | ||
| return "", fmt.Errorf("failed to create sqlite temp dir - %w", err) | ||
| } | ||
| tempFile := filepath.Join(tempDir, "enclave.db") | ||
| return tempFile, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| package migration | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "embed" | ||
| "errors" | ||
| "fmt" | ||
| "io/fs" | ||
| "math/big" | ||
| "sort" | ||
| "strconv" | ||
| "strings" | ||
|
|
||
| gethlog "github.com/ethereum/go-ethereum/log" | ||
| "github.com/ten-protocol/go-ten/go/common/errutil" | ||
| "github.com/ten-protocol/go-ten/go/common/storage/database" | ||
| ) | ||
|
|
||
| const currentMigrationVersionKey = "CURRENT_MIGRATION_VERSION" | ||
|
|
||
| func DBMigration(db *sql.DB, sqlFiles embed.FS, logger gethlog.Logger) error { | ||
| migrationFiles, err := readMigrationFiles(sqlFiles) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| maxMigration := int64(len(migrationFiles)) | ||
|
|
||
| var maxDB int64 | ||
| config, err := database.FetchConfig(db, currentMigrationVersionKey) | ||
| if err != nil { | ||
| // first time there is no entry, so 001 was executed already ( triggered at launch/manifest time ) | ||
| if errors.Is(err, errutil.ErrNotFound) { | ||
| maxDB = 1 | ||
| } else { | ||
| return err | ||
| } | ||
| } else { | ||
| maxDB = ByteArrayToInt(config) | ||
| } | ||
|
|
||
| // write to the database | ||
| for i := maxDB; i < maxMigration; i++ { | ||
| logger.Info("Executing db migration", "file", migrationFiles[i].Name()) | ||
| content, err := sqlFiles.ReadFile(migrationFiles[i].Name()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| err = executeMigration(db, string(content), i) | ||
| if err != nil { | ||
| return fmt.Errorf("unable to execute migration for %s - %w", migrationFiles[i].Name(), err) | ||
| } | ||
| logger.Info("Successfully executed", "file", migrationFiles[i].Name(), "index", i) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func executeMigration(db *sql.DB, content string, migrationOrder int64) error { | ||
| tx, err := db.Begin() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| _, err = tx.Exec(content) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, err = database.WriteConfigToTx(tx, currentMigrationVersionKey, big.NewInt(migrationOrder).Bytes()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return tx.Commit() | ||
| } | ||
|
|
||
| func readMigrationFiles(sqlFiles embed.FS) ([]fs.DirEntry, error) { | ||
| migrationFiles, err := sqlFiles.ReadDir(".") | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // sort the migrationFiles based on the prefix (before "_") | ||
| sort.Slice(migrationFiles, func(i, j int) bool { | ||
| // Extract the number prefix and compare | ||
| return getPrefix(migrationFiles[i]) < getPrefix(migrationFiles[j]) | ||
| }) | ||
|
|
||
| // sanity check the consecutive rule | ||
| for i, file := range migrationFiles { | ||
| prefix := getPrefix(file) | ||
| if i+1 != prefix { | ||
| panic("Invalid migration file. Missing index") | ||
| } | ||
| } | ||
| return migrationFiles, err | ||
| } | ||
|
|
||
| func getPrefix(migrationFile fs.DirEntry) int { | ||
| prefix := strings.Split(migrationFile.Name(), "_")[0] | ||
| number, err := strconv.Atoi(prefix) | ||
| if err != nil { | ||
| panic("Invalid db migration file") | ||
| } | ||
| return number | ||
| } | ||
|
|
||
| func ByteArrayToInt(arr []byte) int64 { | ||
| b := big.NewInt(0) | ||
| b.SetBytes(arr) | ||
| return b.Int64() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.