From eff485b826ddfa6490ea42c179b0f9b860047951 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Fri, 10 May 2024 15:58:36 +0100 Subject: [PATCH] update index approach --- go/enclave/storage/enclavedb/keyvalue.go | 36 +++++++------------ .../storage/init/edgelessdb/001_init.sql | 28 +++++++++------ go/enclave/storage/init/sqlite/001_init.sql | 3 +- 3 files changed, 30 insertions(+), 37 deletions(-) diff --git a/go/enclave/storage/enclavedb/keyvalue.go b/go/enclave/storage/enclavedb/keyvalue.go index 366b098011..336ca37ba7 100644 --- a/go/enclave/storage/enclavedb/keyvalue.go +++ b/go/enclave/storage/enclavedb/keyvalue.go @@ -5,25 +5,24 @@ import ( "database/sql" "errors" "fmt" - "hash/fnv" "github.com/ethereum/go-ethereum/ethdb" "github.com/ten-protocol/go-ten/go/common/errutil" ) const ( - getQry = `select keyvalue.val from keyvalue where keyvalue.ky = ? and keyvalue.ky_full = ?;` + getQry = `select keyvalue.val from keyvalue where keyvalue.ky = ? ;` // `replace` will perform insert or replace if existing and this syntax works for both sqlite and edgeless db - putQry = `replace into keyvalue (ky, ky_full, val) values(?, ?, ?);` - putQryBatch = `replace into keyvalue (ky, ky_full, val) values` - putQryValues = `(?,?,?)` - delQry = `delete from keyvalue where keyvalue.ky = ? and keyvalue.ky_full = ?;` + putQry = `replace into keyvalue (ky, val) values(?, ?);` + putQryBatch = `replace into keyvalue (ky, val) values` + putQryValues = `(?,?)` + delQry = `delete from keyvalue where keyvalue.ky = ? ;` // todo - how is the performance of this? - searchQry = `select ky_full, val from keyvalue where substring(keyvalue.ky_full, 1, ?) = ? and keyvalue.ky_full >= ? order by keyvalue.ky_full asc` + searchQry = `select ky, val from keyvalue where substring(keyvalue.ky, 1, ?) = ? and keyvalue.ky >= ? order by keyvalue.ky asc` ) func Has(ctx context.Context, db *sql.DB, key []byte) (bool, error) { - err := db.QueryRowContext(ctx, getQry, hash(key), key).Scan() + err := db.QueryRowContext(ctx, getQry, key).Scan() if err != nil { if errors.Is(err, sql.ErrNoRows) { return false, nil @@ -36,7 +35,7 @@ func Has(ctx context.Context, db *sql.DB, key []byte) (bool, error) { func Get(ctx context.Context, db *sql.DB, key []byte) ([]byte, error) { var res []byte - err := db.QueryRowContext(ctx, getQry, hash(key), key).Scan(&res) + err := db.QueryRowContext(ctx, getQry, key).Scan(&res) if err != nil { if errors.Is(err, sql.ErrNoRows) { // make sure the error is converted to obscuro-wide not found error @@ -48,7 +47,7 @@ func Get(ctx context.Context, db *sql.DB, key []byte) ([]byte, error) { } func Put(ctx context.Context, db *sql.DB, key []byte, value []byte) error { - _, err := db.ExecContext(ctx, putQry, hash(key), key, value) + _, err := db.ExecContext(ctx, putQry, key, value) return err } @@ -63,7 +62,7 @@ func PutKeyValues(ctx context.Context, tx *sql.Tx, keys [][]byte, vals [][]byte) values := make([]any, 0) for i := range keys { - values = append(values, hash(keys[i]), keys[i], vals[i]) + values = append(values, keys[i], vals[i]) } _, err := tx.ExecContext(ctx, update, values...) if err != nil { @@ -75,13 +74,13 @@ func PutKeyValues(ctx context.Context, tx *sql.Tx, keys [][]byte, vals [][]byte) } func Delete(ctx context.Context, db *sql.DB, key []byte) error { - _, err := db.ExecContext(ctx, delQry, hash(key), key) + _, err := db.ExecContext(ctx, delQry, key) return err } func DeleteKeys(ctx context.Context, db *sql.Tx, keys [][]byte) error { for _, del := range keys { - _, err := db.ExecContext(ctx, delQry, hash(del), del) + _, err := db.ExecContext(ctx, delQry, del) if err != nil { return err } @@ -108,14 +107,3 @@ func NewIterator(ctx context.Context, db *sql.DB, prefix []byte, start []byte) e rows: rows, } } - -// hash returns 4 bytes "hash" of the key to be indexed -// truncating is not sufficient because the keys are not random -func hash(key []byte) []byte { - h := fnv.New32() - _, err := h.Write(key) - if err != nil { - return nil - } - return h.Sum([]byte{}) -} diff --git a/go/enclave/storage/init/edgelessdb/001_init.sql b/go/enclave/storage/init/edgelessdb/001_init.sql index 2c0291b73c..281ee4dc47 100644 --- a/go/enclave/storage/init/edgelessdb/001_init.sql +++ b/go/enclave/storage/init/edgelessdb/001_init.sql @@ -4,11 +4,10 @@ CREATE DATABASE obsdb; create table if not exists obsdb.keyvalue ( id INTEGER AUTO_INCREMENT, - ky binary(4), - ky_full varbinary(64), + ky varbinary(64), val mediumblob NOT NULL, primary key (id), - INDEX (ky) + INDEX USING HASH (ky) ); GRANT ALL ON obsdb.keyvalue TO obscuro; @@ -39,7 +38,7 @@ create table if not exists obsdb.block height int NOT NULL, primary key (id), INDEX (height), - INDEX (hash(4)) + INDEX USING HASH (hash(16)) ); GRANT ALL ON obsdb.block TO obscuro; @@ -64,7 +63,7 @@ create table if not exists obsdb.rollup header blob NOT NULL, compression_block INTEGER NOT NULL, INDEX (compression_block), - INDEX (hash(4)), + INDEX USING HASH (hash(16)), primary key (id) ); GRANT ALL ON obsdb.rollup TO obscuro; @@ -90,7 +89,7 @@ create table if not exists obsdb.batch l1_proof INTEGER, is_executed boolean NOT NULL, primary key (sequence), - INDEX (hash), + INDEX USING HASH (hash(16)), INDEX (body), INDEX (height, is_canonical), INDEX (l1_proof) @@ -107,7 +106,7 @@ create table if not exists obsdb.tx idx int NOT NULL, body int NOT NULL, INDEX (body), - INDEX (hash(4)), + INDEX USING HASH (hash(16)), primary key (id) ); GRANT ALL ON obsdb.tx TO obscuro; @@ -120,7 +119,7 @@ create table if not exists obsdb.exec_tx tx int, batch int NOT NULL, INDEX (batch,tx), - INDEX (created_contract_address(4)), + INDEX (tx, created_contract_address(4)), primary key (id) ); GRANT ALL ON obsdb.exec_tx TO obscuro; @@ -143,8 +142,15 @@ create table if not exists obsdb.events tx int, batch int NOT NULL, INDEX (batch, tx), - INDEX (address(4)), - INDEX (rel_address1(4), rel_address2(4), rel_address3(4), rel_address4(4)), - INDEX (topic0(4), topic1(4), topic2(4), topic3(4), topic4(4)) + INDEX USING HASH (address(16)), + INDEX USING HASH (rel_address1(16)), + INDEX USING HASH (rel_address2(16)), + INDEX USING HASH (rel_address3(16)), + INDEX USING HASH (rel_address4(16)), + INDEX USING HASH (topic0(16)), + INDEX USING HASH (topic1(16)), + INDEX USING HASH (topic2(16)), + INDEX USING HASH (topic3(16)), + INDEX USING HASH (topic4(16)) ); GRANT ALL ON obsdb.events TO obscuro; \ No newline at end of file diff --git a/go/enclave/storage/init/sqlite/001_init.sql b/go/enclave/storage/init/sqlite/001_init.sql index 9e9ea6676e..6244458ad5 100644 --- a/go/enclave/storage/init/sqlite/001_init.sql +++ b/go/enclave/storage/init/sqlite/001_init.sql @@ -1,8 +1,7 @@ create table if not exists keyvalue ( id INTEGER PRIMARY KEY AUTOINCREMENT, - ky binary(4), - ky_full varbinary(64), + ky varbinary(64), val mediumblob NOT NULL ); create index IDX_KV on keyvalue (ky);