From 40afdd5e5b8996e7e9ae6daa694fc310bfa61437 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 18 Oct 2024 18:03:14 -0400 Subject: [PATCH 01/30] wip --- carstore/bs.go | 84 ++++++++--------- carstore/last_shard_cache.go | 67 +++++++++++++ carstore/sqlite_store.go | 178 +++++++++++++++++++++++++++++++++++ 3 files changed, 282 insertions(+), 47 deletions(-) create mode 100644 carstore/last_shard_cache.go create mode 100644 carstore/sqlite_store.go diff --git a/carstore/bs.go b/carstore/bs.go index e7af35d12..7b2f55926 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "sort" - "sync" "sync/atomic" "time" @@ -49,8 +48,11 @@ const MaxSliceLength = 2 << 20 const BigShardThreshold = 2 << 20 type CarStore interface { + // TODO: not really part of general interface CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) + // TODO: not really part of general interface GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) + GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) @@ -65,8 +67,9 @@ type FileCarStore struct { meta *CarStoreGormMeta rootDir string - lscLk sync.Mutex - lastShardCache map[models.Uid]*CarShard + lastShardCache lastShardCache + //lscLk sync.Mutex + //lastShardCache map[models.Uid]*CarShard } func NewCarStore(meta *gorm.DB, root string) (CarStore, error) { @@ -86,15 +89,25 @@ func NewCarStore(meta *gorm.DB, root string) (CarStore, error) { return nil, err } - return &FileCarStore{ - meta: &CarStoreGormMeta{meta: meta}, - rootDir: root, - lastShardCache: make(map[models.Uid]*CarShard), - }, nil + gormMeta := &CarStoreGormMeta{meta: meta} + out := &FileCarStore{ + meta: gormMeta, + rootDir: root, + lastShardCache: lastShardCache{ + source: gormMeta, + }, + } + out.lastShardCache.Init() + return out, nil +} + +type userViewSource interface { + HasUidCid(ctx context.Context, user models.Uid, k cid.Cid) (bool, error) + LookupBlockRef(ctx context.Context, k cid.Cid) (path string, offset int64, user models.Uid, err error) } type userView struct { - cs *FileCarStore + cs userViewSource // TODO: interface-ify, used for .meta.HasUidCid and .meta.LookupBlockRef user models.Uid cache map[cid.Cid]blockformat.Block @@ -108,7 +121,7 @@ func (uv *userView) HashOnRead(hor bool) { } func (uv *userView) Has(ctx context.Context, k cid.Cid) (bool, error) { - return uv.cs.meta.HasUidCid(ctx, uv.user, k) + return uv.cs.HasUidCid(ctx, uv.user, k) } var CacheHits int64 @@ -129,7 +142,7 @@ func (uv *userView) Get(ctx context.Context, k cid.Cid) (blockformat.Block, erro } atomic.AddInt64(&CacheMiss, 1) - path, offset, user, err := uv.cs.meta.LookupBlockRef(ctx, k) + path, offset, user, err := uv.cs.LookupBlockRef(ctx, k) if err != nil { return nil, err } @@ -269,52 +282,25 @@ type DeltaSession struct { baseCid cid.Cid seq int readonly bool - cs *FileCarStore - lastRev string + // cs *FileCarStore // TODO: this is only needed for CloseWithRoot to write back delta session modifications, interface-ify + cs shardWriter + lastRev string } func (cs *FileCarStore) checkLastShardCache(user models.Uid) *CarShard { - cs.lscLk.Lock() - defer cs.lscLk.Unlock() - - ls, ok := cs.lastShardCache[user] - if ok { - return ls - } - - return nil + return cs.lastShardCache.check(user) } func (cs *FileCarStore) removeLastShardCache(user models.Uid) { - cs.lscLk.Lock() - defer cs.lscLk.Unlock() - - delete(cs.lastShardCache, user) + cs.lastShardCache.remove(user) } func (cs *FileCarStore) putLastShardCache(ls *CarShard) { - cs.lscLk.Lock() - defer cs.lscLk.Unlock() - - cs.lastShardCache[ls.Usr] = ls + cs.lastShardCache.put(ls) } func (cs *FileCarStore) getLastShard(ctx context.Context, user models.Uid) (*CarShard, error) { - ctx, span := otel.Tracer("carstore").Start(ctx, "getLastShard") - defer span.End() - - maybeLs := cs.checkLastShardCache(user) - if maybeLs != nil { - return maybeLs, nil - } - - lastShard, err := cs.meta.GetLastShard(ctx, user) - if err != nil { - return nil, err - } - - cs.putLastShardCache(lastShard) - return lastShard, nil + return cs.lastShardCache.get(ctx, user) } var ErrRepoBaseMismatch = fmt.Errorf("attempted a delta session on top of the wrong previous head") @@ -339,7 +325,7 @@ func (cs *FileCarStore) NewDeltaSession(ctx context.Context, user models.Uid, si blks: make(map[cid.Cid]blockformat.Block), base: &userView{ user: user, - cs: cs, + cs: cs.meta, prefetch: true, cache: make(map[cid.Cid]blockformat.Block), }, @@ -355,7 +341,7 @@ func (cs *FileCarStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) return &DeltaSession{ base: &userView{ user: user, - cs: cs, + cs: cs.meta, prefetch: false, cache: make(map[cid.Cid]blockformat.Block), }, @@ -600,6 +586,10 @@ func WriteCarHeader(w io.Writer, root cid.Cid) (int64, error) { return hnw, nil } +type shardWriter interface { + writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) +} + func (cs *FileCarStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { buf := new(bytes.Buffer) diff --git a/carstore/last_shard_cache.go b/carstore/last_shard_cache.go new file mode 100644 index 000000000..b0022ef6d --- /dev/null +++ b/carstore/last_shard_cache.go @@ -0,0 +1,67 @@ +package carstore + +import ( + "context" + "github.com/bluesky-social/indigo/models" + "go.opentelemetry.io/otel" + "sync" +) + +type LastShardSource interface { + GetLastShard(context.Context, models.Uid) (*CarShard, error) +} + +type lastShardCache struct { + source LastShardSource + + lscLk sync.Mutex + lastShardCache map[models.Uid]*CarShard +} + +func (lsc *lastShardCache) Init() { + lsc.lastShardCache = make(map[models.Uid]*CarShard) +} + +func (lsc *lastShardCache) check(user models.Uid) *CarShard { + lsc.lscLk.Lock() + defer lsc.lscLk.Unlock() + + ls, ok := lsc.lastShardCache[user] + if ok { + return ls + } + + return nil +} + +func (lsc *lastShardCache) remove(user models.Uid) { + lsc.lscLk.Lock() + defer lsc.lscLk.Unlock() + + delete(lsc.lastShardCache, user) +} + +func (lsc *lastShardCache) put(ls *CarShard) { + lsc.lscLk.Lock() + defer lsc.lscLk.Unlock() + + lsc.lastShardCache[ls.Usr] = ls +} + +func (lsc *lastShardCache) get(ctx context.Context, user models.Uid) (*CarShard, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "getLastShard") + defer span.End() + + maybeLs := lsc.check(user) + if maybeLs != nil { + return maybeLs, nil + } + + lastShard, err := lsc.source.GetLastShard(ctx, user) + if err != nil { + return nil, err + } + + lsc.put(lastShard) + return lastShard, nil +} diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go new file mode 100644 index 000000000..b4ac22d79 --- /dev/null +++ b/carstore/sqlite_store.go @@ -0,0 +1,178 @@ +package carstore + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "github.com/bluesky-social/indigo/models" + blockformat "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipld/go-car" + _ "github.com/mattn/go-sqlite3" + "go.opentelemetry.io/otel" + "io" +) + +//var log = logging.Logger("sqstore") + +type SQLiteStore struct { + db *sql.DB + + lastShardCache lastShardCache +} + +func (sqs *SQLiteStore) Open(path string) error { + db, err := sql.Open("sqlite3", path) + if err != nil { + return err + } + sqs.db = db + sqs.lastShardCache.source = sqs + return nil +} + +// writeNewShard needed for DeltaSession.CloseWithRoot +func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { + //TODO implement me + panic("implement me") +} + +// GetLastShard nedeed for NewDeltaSession indirectly through lastShardCache +func (sqs *SQLiteStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarShard, error) { + //TODO implement me + panic("implement me") +} + +func (sqs *SQLiteStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { + //TODO remove from CarStore interface + panic("implement me") +} + +func (sqs *SQLiteStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { + //TODO remove from CarStore interface + return nil, nil +} + +func (sqs *SQLiteStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { + //TODO implement me + panic("implement me") +} + +func (sqs *SQLiteStore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { + //TODO implement me + panic("implement me") +} + +func (sqs *SQLiteStore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { + // TODO: same as FileCarStore, re-unify + ctx, span := otel.Tracer("carstore").Start(ctx, "ImportSlice") + defer span.End() + + carr, err := car.NewCarReader(bytes.NewReader(carslice)) + if err != nil { + return cid.Undef, nil, err + } + + if len(carr.Header.Roots) != 1 { + return cid.Undef, nil, fmt.Errorf("invalid car file, header must have a single root (has %d)", len(carr.Header.Roots)) + } + + ds, err := sqs.NewDeltaSession(ctx, uid, since) + if err != nil { + return cid.Undef, nil, fmt.Errorf("new delta session failed: %w", err) + } + + var cids []cid.Cid + for { + blk, err := carr.Next() + if err != nil { + if err == io.EOF { + break + } + return cid.Undef, nil, err + } + + cids = append(cids, blk.Cid()) + + if err := ds.Put(ctx, blk); err != nil { + return cid.Undef, nil, err + } + } + + return carr.Header.Roots[0], ds, nil +} + +func (sqs *SQLiteStore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession") + defer span.End() + + // TODO: ensure that we don't write updates on top of the wrong head + // this needs to be a compare and swap type operation + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return nil, err + } + + if since != nil && *since != lastShard.Rev { + return nil, fmt.Errorf("revision mismatch: %s != %s: %w", *since, lastShard.Rev, ErrRepoBaseMismatch) + } + + return &DeltaSession{ + fresh: blockstore.NewBlockstore(datastore.NewMapDatastore()), + blks: make(map[cid.Cid]blockformat.Block), + base: &userView{ + user: user, + cs: sqs, + prefetch: true, + cache: make(map[cid.Cid]blockformat.Block), + }, + user: user, + baseCid: lastShard.Root.CID, + cs: sqs, + seq: lastShard.Seq + 1, + lastRev: lastShard.Rev, + }, nil +} + +func (sqs *SQLiteStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { + //TODO implement me + panic("implement me") +} + +func (sqs *SQLiteStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error { + //TODO implement me + panic("implement me") +} + +func (sqs *SQLiteStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { + //TODO implement me + panic("implement me") +} + +func (sqs *SQLiteStore) WipeUserData(ctx context.Context, user models.Uid) error { + //TODO implement me + panic("implement me") +} + +// HasUidCid needed for NewDeltaSession userView +func (sqs *SQLiteStore) HasUidCid(ctx context.Context, user models.Uid, k cid.Cid) (bool, error) { + //TODO implement me + panic("implement me") +} + +// LookupBlockRef needed for NewDeltaSession userView +func (sqs *SQLiteStore) LookupBlockRef(ctx context.Context, k cid.Cid) (path string, offset int64, user models.Uid, err error) { + //TODO implement me + panic("implement me") +} + +func (sqs *SQLiteStore) CarStore() CarStore { + return sqs +} + +func (sqs *SQLiteStore) Close() { + sqs.db.Close() +} From 450b771a17788587a8db4b208c2f2615d32bb805 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 25 Oct 2024 14:43:13 -0400 Subject: [PATCH 02/30] wip: new storage skeleton --- carstore/sqlite_store.go | 87 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index b4ac22d79..c8b1c5313 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -36,6 +36,76 @@ func (sqs *SQLiteStore) Open(path string) error { // writeNewShard needed for DeltaSession.CloseWithRoot func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { + // write a bunch of (uid,cid,block) + + insertStatement, err := sqs.db.PrepareContext(ctx, "INSERT INTO blocks (uid, cid, block) VALUES (?, ?, ?)") + if err != nil { + return nil, err + } + for cid, block := range blks { + _, err = insertStatement.Exec(user, cid, block.RawData()) + if err != nil { + return nil, err + } + } + + buf := new(bytes.Buffer) + hnw, err := WriteCarHeader(buf, root) + if err != nil { + return nil, fmt.Errorf("failed to write car header: %w", err) + } + + // TODO: writing these blocks in map traversal order is bad, I believe the + // optimal ordering will be something like reverse-write-order, but random + // is definitely not it + + offset := hnw + //brefs := make([]*blockRef, 0, len(ds.blks)) + brefs := make([]map[string]interface{}, 0, len(blks)) + for k, blk := range blks { + nw, err := LdWrite(buf, k.Bytes(), blk.RawData()) + if err != nil { + return nil, fmt.Errorf("failed to write block: %w", err) + } + + /* + brefs = append(brefs, &blockRef{ + Cid: k.String(), + Offset: offset, + Shard: shard.ID, + }) + */ + // adding things to the db by map is the only way to get gorm to not + // add the 'returning' clause, which costs a lot of time + brefs = append(brefs, map[string]interface{}{ + "cid": models.DbCID{CID: k}, + "offset": offset, + }) + + offset += nw + } + + path, err := cs.writeNewShardFile(ctx, user, seq, buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to write shard file: %w", err) + } + + shard := CarShard{ + Root: models.DbCID{CID: root}, + DataStart: hnw, + Seq: seq, + Path: path, + Usr: user, + Rev: rev, + } + + if err := cs.putShard(ctx, &shard, brefs, rmcids); err != nil { + return nil, err + } + + sqs.lastShardCache.put(&shard) + + return buf.Bytes(), nil //TODO implement me panic("implement me") } @@ -138,8 +208,17 @@ func (sqs *SQLiteStore) NewDeltaSession(ctx context.Context, user models.Uid, si } func (sqs *SQLiteStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { - //TODO implement me - panic("implement me") + return &DeltaSession{ + base: &userView{ + user: user, + cs: sqs, + prefetch: false, + cache: make(map[cid.Cid]blockformat.Block), + }, + readonly: true, + user: user, + cs: sqs, + }, nil } func (sqs *SQLiteStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error { @@ -173,6 +252,6 @@ func (sqs *SQLiteStore) CarStore() CarStore { return sqs } -func (sqs *SQLiteStore) Close() { - sqs.db.Close() +func (sqs *SQLiteStore) Close() error { + return sqs.db.Close() } From 2e408c1dc2e98e0dc90949408bed4025e002babf Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Thu, 31 Oct 2024 21:37:06 -0400 Subject: [PATCH 03/30] wip: starting to fail at unit testing new carstore --- carstore/bs.go | 20 +- carstore/last_shard_cache.go | 3 + carstore/repo_test.go | 357 +++++++++++++++++++---------------- carstore/sqlite_store.go | 313 ++++++++++++++++++++++-------- 4 files changed, 444 insertions(+), 249 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index 7b2f55926..2e874f6da 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -68,8 +68,6 @@ type FileCarStore struct { rootDir string lastShardCache lastShardCache - //lscLk sync.Mutex - //lastShardCache map[models.Uid]*CarShard } func NewCarStore(meta *gorm.DB, root string) (CarStore, error) { @@ -273,11 +271,19 @@ func (uv *userView) GetSize(ctx context.Context, k cid.Cid) (int, error) { return len(blk.RawData()), nil } +// subset of blockstore.Blockstore that we actually use here +type minBlockstore interface { + Get(ctx context.Context, bcid cid.Cid) (blockformat.Block, error) + Has(ctx context.Context, bcid cid.Cid) (bool, error) + GetSize(ctx context.Context, bcid cid.Cid) (int, error) +} + type DeltaSession struct { - fresh blockstore.Blockstore - blks map[cid.Cid]blockformat.Block - rmcids map[cid.Cid]bool - base blockstore.Blockstore + fresh blockstore.Blockstore + blks map[cid.Cid]blockformat.Block + rmcids map[cid.Cid]bool + //base blockstore.Blockstore + base minBlockstore user models.Uid baseCid cid.Cid seq int @@ -365,7 +371,6 @@ func (cs *FileCarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR } } - // TODO: Why does ReadUserCar want shards seq DESC but CompactUserShards wants seq ASC ? shards, err := cs.meta.GetUserShardsDesc(ctx, user, earlySeq) if err != nil { return err @@ -587,6 +592,7 @@ func WriteCarHeader(w io.Writer, root cid.Cid) (int64, error) { } type shardWriter interface { + // writeNewShard stores blocks in `blks` arg and creates a new shard to propagate out to our firehose writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) } diff --git a/carstore/last_shard_cache.go b/carstore/last_shard_cache.go index b0022ef6d..8371b8883 100644 --- a/carstore/last_shard_cache.go +++ b/carstore/last_shard_cache.go @@ -42,6 +42,9 @@ func (lsc *lastShardCache) remove(user models.Uid) { } func (lsc *lastShardCache) put(ls *CarShard) { + if ls == nil { + return + } lsc.lscLk.Lock() defer lsc.lscLk.Unlock() diff --git a/carstore/repo_test.go b/carstore/repo_test.go index a4d2c8cb8..bdddda424 100644 --- a/carstore/repo_test.go +++ b/carstore/repo_test.go @@ -55,6 +55,22 @@ func testCarStore() (CarStore, func(), error) { }, nil } +func testSqliteCarStore() (CarStore, func(), error) { + sqs := &SQLiteStore{} + err := sqs.Open(":memory:") + if err != nil { + return nil, nil, err + } + return sqs, func() {}, nil +} + +type testFactory func() (CarStore, func(), error) + +var backends = map[string]testFactory{ + "cartore": testCarStore, + "sqlite": testSqliteCarStore, +} + func testFlatfsBs() (blockstore.Blockstore, func(), error) { tempdir, err := os.MkdirTemp("", "msttest-") if err != nil { @@ -73,85 +89,90 @@ func testFlatfsBs() (blockstore.Blockstore, func(), error) { }, nil } -func TestBasicOperation(t *testing.T) { +func TestBasicOperation(ot *testing.T) { ctx := context.TODO() - cs, cleanup, err := testCarStore() - if err != nil { - t.Fatal(err) - } - defer cleanup() + for fname, tf := range backends { + ot.Run(fname, func(t *testing.T) { - ds, err := cs.NewDeltaSession(ctx, 1, nil) - if err != nil { - t.Fatal(err) - } + cs, cleanup, err := tf() + if err != nil { + t.Fatal(err) + } + defer cleanup() - ncid, rev, err := setupRepo(ctx, ds, false) - if err != nil { - t.Fatal(err) - } + ds, err := cs.NewDeltaSession(ctx, 1, nil) + if err != nil { + t.Fatal(err) + } - if _, err := ds.CloseWithRoot(ctx, ncid, rev); err != nil { - t.Fatal(err) - } + ncid, rev, err := setupRepo(ctx, ds, false) + if err != nil { + t.Fatal(err) + } - var recs []cid.Cid - head := ncid - for i := 0; i < 10; i++ { - ds, err := cs.NewDeltaSession(ctx, 1, &rev) - if err != nil { - t.Fatal(err) - } + if _, err := ds.CloseWithRoot(ctx, ncid, rev); err != nil { + t.Fatal(err) + } - rr, err := repo.OpenRepo(ctx, ds, head) - if err != nil { - t.Fatal(err) - } + var recs []cid.Cid + head := ncid + for i := 0; i < 10; i++ { + ds, err := cs.NewDeltaSession(ctx, 1, &rev) + if err != nil { + t.Fatal(err) + } - rc, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ - Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), - }) - if err != nil { - t.Fatal(err) - } + rr, err := repo.OpenRepo(ctx, ds, head) + if err != nil { + t.Fatal(err) + } + + rc, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ + Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatal(err) + } - recs = append(recs, rc) + recs = append(recs, rc) - kmgr := &util.FakeKeyManager{} - nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) - if err != nil { - t.Fatal(err) - } + kmgr := &util.FakeKeyManager{} + nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) + if err != nil { + t.Fatal(err) + } - rev = nrev + rev = nrev - if err := ds.CalcDiff(ctx, nil); err != nil { - t.Fatal(err) - } + if err := ds.CalcDiff(ctx, nil); err != nil { + t.Fatal(err) + } - if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { - t.Fatal(err) - } + if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { + t.Fatal(err) + } - head = nroot - } + head = nroot + } - buf := new(bytes.Buffer) - if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { - t.Fatal(err) - } - checkRepo(t, cs, buf, recs) + buf := new(bytes.Buffer) + if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { + t.Fatal(err) + } + checkRepo(t, cs, buf, recs) - if _, err := cs.CompactUserShards(ctx, 1, false); err != nil { - t.Fatal(err) - } + if _, err := cs.CompactUserShards(ctx, 1, false); err != nil { + t.Fatal(err) + } - buf = new(bytes.Buffer) - if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { - t.Fatal(err) + buf = new(bytes.Buffer) + if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { + t.Fatal(err) + } + checkRepo(t, cs, buf, recs) + }) } - checkRepo(t, cs, buf, recs) } func TestRepeatedCompactions(t *testing.T) { @@ -319,6 +340,15 @@ func BenchmarkRepoWritesCarstore(b *testing.B) { ctx := context.TODO() cs, cleanup, err := testCarStore() + innerBenchmarkRepoWritesCarstore(b, ctx, cs, cleanup, err) +} +func BenchmarkRepoWritesSqliteCarstore(b *testing.B) { + ctx := context.TODO() + + cs, cleanup, err := testSqliteCarStore() + innerBenchmarkRepoWritesCarstore(b, ctx, cs, cleanup, err) +} +func innerBenchmarkRepoWritesCarstore(b *testing.B, ctx context.Context, cs CarStore, cleanup func(), err error) { if err != nil { b.Fatal(err) } @@ -453,131 +483,136 @@ func BenchmarkRepoWritesSqlite(b *testing.B) { } } -func TestDuplicateBlockAcrossShards(t *testing.T) { +func TestDuplicateBlockAcrossShards(ot *testing.T) { ctx := context.TODO() - cs, cleanup, err := testCarStore() - if err != nil { - t.Fatal(err) - } - defer cleanup() + for fname, tf := range backends { + ot.Run(fname, func(t *testing.T) { - ds1, err := cs.NewDeltaSession(ctx, 1, nil) - if err != nil { - t.Fatal(err) - } + cs, cleanup, err := tf() + if err != nil { + t.Fatal(err) + } + defer cleanup() - ds2, err := cs.NewDeltaSession(ctx, 2, nil) - if err != nil { - t.Fatal(err) - } + ds1, err := cs.NewDeltaSession(ctx, 1, nil) + if err != nil { + t.Fatal(err) + } - ds3, err := cs.NewDeltaSession(ctx, 3, nil) - if err != nil { - t.Fatal(err) - } + ds2, err := cs.NewDeltaSession(ctx, 2, nil) + if err != nil { + t.Fatal(err) + } - var cids []cid.Cid - var revs []string - for _, ds := range []*DeltaSession{ds1, ds2, ds3} { - ncid, rev, err := setupRepo(ctx, ds, true) - if err != nil { - t.Fatal(err) - } + ds3, err := cs.NewDeltaSession(ctx, 3, nil) + if err != nil { + t.Fatal(err) + } - if _, err := ds.CloseWithRoot(ctx, ncid, rev); err != nil { - t.Fatal(err) - } - cids = append(cids, ncid) - revs = append(revs, rev) - } + var cids []cid.Cid + var revs []string + for _, ds := range []*DeltaSession{ds1, ds2, ds3} { + ncid, rev, err := setupRepo(ctx, ds, true) + if err != nil { + t.Fatal(err) + } - var recs []cid.Cid - head := cids[1] - rev := revs[1] - for i := 0; i < 10; i++ { - ds, err := cs.NewDeltaSession(ctx, 2, &rev) - if err != nil { - t.Fatal(err) - } + if _, err := ds.CloseWithRoot(ctx, ncid, rev); err != nil { + t.Fatal(err) + } + cids = append(cids, ncid) + revs = append(revs, rev) + } - rr, err := repo.OpenRepo(ctx, ds, head) - if err != nil { - t.Fatal(err) - } + var recs []cid.Cid + head := cids[1] + rev := revs[1] + for i := 0; i < 10; i++ { + ds, err := cs.NewDeltaSession(ctx, 2, &rev) + if err != nil { + t.Fatal(err) + } - rc, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ - Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), - }) - if err != nil { - t.Fatal(err) - } + rr, err := repo.OpenRepo(ctx, ds, head) + if err != nil { + t.Fatal(err) + } - recs = append(recs, rc) + rc, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ + Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatal(err) + } - kmgr := &util.FakeKeyManager{} - nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) - if err != nil { - t.Fatal(err) - } + recs = append(recs, rc) - rev = nrev + kmgr := &util.FakeKeyManager{} + nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) + if err != nil { + t.Fatal(err) + } - if err := ds.CalcDiff(ctx, nil); err != nil { - t.Fatal(err) - } + rev = nrev - if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { - t.Fatal(err) - } + if err := ds.CalcDiff(ctx, nil); err != nil { + t.Fatal(err) + } - head = nroot - } + if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { + t.Fatal(err) + } - // explicitly update the profile object - { - ds, err := cs.NewDeltaSession(ctx, 2, &rev) - if err != nil { - t.Fatal(err) - } + head = nroot + } - rr, err := repo.OpenRepo(ctx, ds, head) - if err != nil { - t.Fatal(err) - } + // explicitly update the profile object + { + ds, err := cs.NewDeltaSession(ctx, 2, &rev) + if err != nil { + t.Fatal(err) + } - desc := "this is so unique" - rc, err := rr.UpdateRecord(ctx, "app.bsky.actor.profile/self", &appbsky.ActorProfile{ - Description: &desc, - }) - if err != nil { - t.Fatal(err) - } + rr, err := repo.OpenRepo(ctx, ds, head) + if err != nil { + t.Fatal(err) + } + + desc := "this is so unique" + rc, err := rr.UpdateRecord(ctx, "app.bsky.actor.profile/self", &appbsky.ActorProfile{ + Description: &desc, + }) + if err != nil { + t.Fatal(err) + } - recs = append(recs, rc) + recs = append(recs, rc) - kmgr := &util.FakeKeyManager{} - nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) - if err != nil { - t.Fatal(err) - } + kmgr := &util.FakeKeyManager{} + nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) + if err != nil { + t.Fatal(err) + } - rev = nrev + rev = nrev - if err := ds.CalcDiff(ctx, nil); err != nil { - t.Fatal(err) - } + if err := ds.CalcDiff(ctx, nil); err != nil { + t.Fatal(err) + } - if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { - t.Fatal(err) - } + if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { + t.Fatal(err) + } - head = nroot - } + head = nroot + } - buf := new(bytes.Buffer) - if err := cs.ReadUserCar(ctx, 2, "", true, buf); err != nil { - t.Fatal(err) + buf := new(bytes.Buffer) + if err := cs.ReadUserCar(ctx, 2, "", true, buf); err != nil { + t.Fatal(err) + } + checkRepo(t, cs, buf, recs) + }) } - checkRepo(t, cs, buf, recs) } diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index c8b1c5313..73ee9c797 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -4,16 +4,21 @@ import ( "bytes" "context" "database/sql" + "errors" "fmt" "github.com/bluesky-social/indigo/models" blockformat "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-libipfs/blocks" "github.com/ipld/go-car" _ "github.com/mattn/go-sqlite3" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "go.opentelemetry.io/otel" "io" + "log/slog" ) //var log = logging.Logger("sqstore") @@ -27,113 +32,130 @@ type SQLiteStore struct { func (sqs *SQLiteStore) Open(path string) error { db, err := sql.Open("sqlite3", path) if err != nil { - return err + return fmt.Errorf("%s: sqlite could not open, %w", path, err) + } + _, err = db.Exec("CREATE TABLE IF NOT EXISTS blocks (uid int, cid blob, rev varchar, root varchar, block blob, PRIMARY KEY(uid,cid))") + if err != nil { + return fmt.Errorf("%s: create table blocks..., %w", path, err) } sqs.db = db sqs.lastShardCache.source = sqs + sqs.lastShardCache.Init() return nil } // writeNewShard needed for DeltaSession.CloseWithRoot func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { - // write a bunch of (uid,cid,block) - - insertStatement, err := sqs.db.PrepareContext(ctx, "INSERT INTO blocks (uid, cid, block) VALUES (?, ?, ?)") - if err != nil { - return nil, err - } - for cid, block := range blks { - _, err = insertStatement.Exec(user, cid, block.RawData()) - if err != nil { - return nil, err - } - } - + sqWriteNewShard.Inc() + // TODO: trace span here + // this is "write many blocks", "write one block" is above in putBlock(). keep them in sync. buf := new(bytes.Buffer) hnw, err := WriteCarHeader(buf, root) if err != nil { return nil, fmt.Errorf("failed to write car header: %w", err) } + offset := hnw - // TODO: writing these blocks in map traversal order is bad, I believe the - // optimal ordering will be something like reverse-write-order, but random - // is definitely not it + insertStatement, err := sqs.db.PrepareContext(ctx, "INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?)") + if err != nil { + return nil, fmt.Errorf("bad block insert sql, %w", err) + } + defer insertStatement.Close() - offset := hnw - //brefs := make([]*blockRef, 0, len(ds.blks)) - brefs := make([]map[string]interface{}, 0, len(blks)) - for k, blk := range blks { - nw, err := LdWrite(buf, k.Bytes(), blk.RawData()) + dbroot := models.DbCID{root} + + for bcid, block := range blks { + // build shard for output firehose + nw, err := LdWrite(buf, bcid.Bytes(), block.RawData()) if err != nil { return nil, fmt.Errorf("failed to write block: %w", err) } - - /* - brefs = append(brefs, &blockRef{ - Cid: k.String(), - Offset: offset, - Shard: shard.ID, - }) - */ - // adding things to the db by map is the only way to get gorm to not - // add the 'returning' clause, which costs a lot of time - brefs = append(brefs, map[string]interface{}{ - "cid": models.DbCID{CID: k}, - "offset": offset, - }) - offset += nw - } - path, err := cs.writeNewShardFile(ctx, user, seq, buf.Bytes()) - if err != nil { - return nil, fmt.Errorf("failed to write shard file: %w", err) + // TODO: better databases have an insert-many option for a prepared statement + dbcid := models.DbCID{CID: bcid} + _, err = insertStatement.ExecContext(ctx, user, dbcid, rev, dbroot, block.RawData()) + if err != nil { + return nil, fmt.Errorf("(uid,cid) block store failed, %w", err) + } } shard := CarShard{ Root: models.DbCID{CID: root}, DataStart: hnw, Seq: seq, - Path: path, Usr: user, Rev: rev, } - if err := cs.putShard(ctx, &shard, brefs, rmcids); err != nil { - return nil, err - } - sqs.lastShardCache.put(&shard) return buf.Bytes(), nil - //TODO implement me - panic("implement me") } +var ErrNothingThere = errors.New("nothing to read)") + // GetLastShard nedeed for NewDeltaSession indirectly through lastShardCache +// What we actually seem to need from this: last {Rev, Root.CID} func (sqs *SQLiteStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarShard, error) { - //TODO implement me - panic("implement me") + sqGetLastShard.Inc() + qstmt, err := sqs.db.PrepareContext(ctx, "SELECT rev, root FROM blocks WHERE uid = ? ORDER BY rev DESC LIMIT 1") + if err != nil { + return nil, fmt.Errorf("bad last shard sql, %w", err) + } + rows, err := qstmt.QueryContext(ctx, uid) + if err != nil { + return nil, fmt.Errorf("last shard err, %w", err) + } + if rows.Next() { + var rev string + var rootb models.DbCID + err = rows.Scan(&rev, &rootb) + if err != nil { + return nil, fmt.Errorf("last shard bad scan, %w", err) + } + return &CarShard{ + Root: rootb, + Rev: rev, + }, nil + } + return nil, nil } func (sqs *SQLiteStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { - //TODO remove from CarStore interface - panic("implement me") + slog.Warn("TODO: don't call compaction") + return nil, nil } func (sqs *SQLiteStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { - //TODO remove from CarStore interface + slog.Warn("TODO: don't call compaction targets") return nil, nil } func (sqs *SQLiteStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { - //TODO implement me - panic("implement me") + // TODO: same as FileCarStore; re-unify + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return cid.Undef, err + } + if lastShard.ID == 0 { + return cid.Undef, nil + } + + return lastShard.Root.CID, nil } func (sqs *SQLiteStore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { - //TODO implement me - panic("implement me") + // TODO: same as FileCarStore; re-unify + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return "", err + } + if lastShard.ID == 0 { + return "", nil + } + + return lastShard.Rev, nil } func (sqs *SQLiteStore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { @@ -175,6 +197,8 @@ func (sqs *SQLiteStore) ImportSlice(ctx context.Context, uid models.Uid, since * return carr.Header.Roots[0], ds, nil } +var zeroShard CarShard + func (sqs *SQLiteStore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession") defer span.End() @@ -183,7 +207,11 @@ func (sqs *SQLiteStore) NewDeltaSession(ctx context.Context, user models.Uid, si // this needs to be a compare and swap type operation lastShard, err := sqs.lastShardCache.get(ctx, user) if err != nil { - return nil, err + return nil, fmt.Errorf("NewDeltaSession, lsc, %w", err) + } + + if lastShard == nil { + lastShard = &zeroShard } if since != nil && *since != lastShard.Rev { @@ -193,11 +221,9 @@ func (sqs *SQLiteStore) NewDeltaSession(ctx context.Context, user models.Uid, si return &DeltaSession{ fresh: blockstore.NewBlockstore(datastore.NewMapDatastore()), blks: make(map[cid.Cid]blockformat.Block), - base: &userView{ - user: user, - cs: sqs, - prefetch: true, - cache: make(map[cid.Cid]blockformat.Block), + base: &sqliteUserView{ + uid: user, + sqs: sqs, }, user: user, baseCid: lastShard.Root.CID, @@ -209,11 +235,9 @@ func (sqs *SQLiteStore) NewDeltaSession(ctx context.Context, user models.Uid, si func (sqs *SQLiteStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { return &DeltaSession{ - base: &userView{ - user: user, - cs: sqs, - prefetch: false, - cache: make(map[cid.Cid]blockformat.Block), + base: &sqliteUserView{ + uid: user, + sqs: sqs, }, readonly: true, user: user, @@ -221,31 +245,56 @@ func (sqs *SQLiteStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) }, nil } +// ReadUserCar +// incremental is only ever called true func (sqs *SQLiteStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error { //TODO implement me + // TODO: get help understanding what PDS does for this panic("implement me") } +// Stat is only used in a debugging admin handler +// don't bother implementing it (for now?) func (sqs *SQLiteStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { - //TODO implement me - panic("implement me") + slog.Warn("Stat debugging method not implemented for sqlite store") + return nil, nil } func (sqs *SQLiteStore) WipeUserData(ctx context.Context, user models.Uid) error { - //TODO implement me - panic("implement me") + deleteResult, err := sqs.db.ExecContext(ctx, "DELETE FROM blocks WHERE uid = ?", user) + nrows, ierr := deleteResult.RowsAffected() + if ierr == nil { + sqRowsDeleted.Add(float64(nrows)) + } + if err == nil { + err = ierr + } + return err } // HasUidCid needed for NewDeltaSession userView -func (sqs *SQLiteStore) HasUidCid(ctx context.Context, user models.Uid, k cid.Cid) (bool, error) { - //TODO implement me - panic("implement me") -} - -// LookupBlockRef needed for NewDeltaSession userView -func (sqs *SQLiteStore) LookupBlockRef(ctx context.Context, k cid.Cid) (path string, offset int64, user models.Uid, err error) { - //TODO implement me - panic("implement me") +func (sqs *SQLiteStore) HasUidCid(ctx context.Context, user models.Uid, bcid cid.Cid) (bool, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + sqHas.Inc() + qstmt, err := sqs.db.PrepareContext(ctx, "SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") + if err != nil { + return false, fmt.Errorf("hasUC sql, %w", err) + } + defer qstmt.Close() + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{bcid}) + if err != nil { + return false, fmt.Errorf("hasUC err, %w", err) + } + if rows.Next() { + var rev string + var rootb models.DbCID + err = rows.Scan(&rev, &rootb) + if err != nil { + return false, fmt.Errorf("hasUC bad scan, %w", err) + } + return true, nil + } + return false, nil } func (sqs *SQLiteStore) CarStore() CarStore { @@ -255,3 +304,105 @@ func (sqs *SQLiteStore) CarStore() CarStore { func (sqs *SQLiteStore) Close() error { return sqs.db.Close() } + +func (sqs *SQLiteStore) getBlock(ctx context.Context, user models.Uid, bcid cid.Cid) (blockformat.Block, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + sqGetBlock.Inc() + qstmt, err := sqs.db.PrepareContext(ctx, "SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") + defer qstmt.Close() + if err != nil { + return nil, fmt.Errorf("hasUC sql, %w", err) + } + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{bcid}) + if err != nil { + return nil, fmt.Errorf("hasUC err, %w", err) + } + if rows.Next() { + //var rev string + //var rootb models.DbCID + var blockb []byte + err = rows.Scan(&blockb) + if err != nil { + return nil, fmt.Errorf("hasUC bad scan, %w", err) + } + return blocks.NewBlock(blockb), nil + } + return nil, ErrNothingThere +} + +func (sqs *SQLiteStore) getBlockSize(ctx context.Context, user models.Uid, bcid cid.Cid) (int64, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + sqGetBlockSize.Inc() + qstmt, err := sqs.db.PrepareContext(ctx, "SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") + if err != nil { + return 0, fmt.Errorf("hasUC sql, %w", err) + } + defer qstmt.Close() + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{bcid}) + if err != nil { + return 0, fmt.Errorf("hasUC err, %w", err) + } + if rows.Next() { + var out int64 + err = rows.Scan(&out) + if err != nil { + return 0, fmt.Errorf("hasUC bad scan, %w", err) + } + return out, nil + } + return 0, nil +} + +type sqliteUserView struct { + sqs *SQLiteStore + uid models.Uid +} + +func (s sqliteUserView) Has(ctx context.Context, c cid.Cid) (bool, error) { + // TODO: cache block metadata? + return s.sqs.HasUidCid(ctx, s.uid, c) +} + +func (s sqliteUserView) Get(ctx context.Context, c cid.Cid) (blockformat.Block, error) { + // TODO: cache blocks? + return s.sqs.getBlock(ctx, s.uid, c) +} + +func (s sqliteUserView) GetSize(ctx context.Context, c cid.Cid) (int, error) { + // TODO: cache block metadata? + bigsize, err := s.sqs.getBlockSize(ctx, s.uid, c) + return int(bigsize), err +} + +// ensure we implement the interface +var _ minBlockstore = (*sqliteUserView)(nil) + +var sqRowsDeleted = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_rows_deleted", + Help: "User rows deleted in sqlite backend", +}) + +var sqGetBlock = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_get_block", + Help: "get block sqlite backend", +}) + +var sqGetBlockSize = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_get_block_size", + Help: "get block size sqlite backend", +}) + +var sqHas = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_has", + Help: "check block presence sqlite backend", +}) + +var sqGetLastShard = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_get_last_shard", + Help: "get last shard sqlite backend", +}) + +var sqWriteNewShard = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_write_shard", + Help: "write shard blocks sqlite backend", +}) From 79252d7caac34af73109500eb45db165a9f41e1f Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 1 Nov 2024 16:25:22 -0400 Subject: [PATCH 04/30] passes unit tests --- carstore/bs.go | 10 +-- carstore/repo_test.go | 34 +++++++--- carstore/sqlite_store.go | 141 ++++++++++++++++++++++++++++++++------- repo/repo.go | 8 +-- 4 files changed, 151 insertions(+), 42 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index 2e874f6da..8657d2ded 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -358,7 +358,7 @@ func (cs *FileCarStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) } // TODO: incremental is only ever called true, remove the param -func (cs *FileCarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error { +func (cs *FileCarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, shardOut io.Writer) error { ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") defer span.End() @@ -390,12 +390,12 @@ func (cs *FileCarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR if err := car.WriteHeader(&car.CarHeader{ Roots: []cid.Cid{shards[0].Root.CID}, Version: 1, - }, w); err != nil { + }, shardOut); err != nil { return err } for _, sh := range shards { - if err := cs.writeShardBlocks(ctx, &sh, w); err != nil { + if err := cs.writeShardBlocks(ctx, &sh, shardOut); err != nil { return err } } @@ -405,7 +405,7 @@ func (cs *FileCarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR // inner loop part of ReadUserCar // copy shard blocks from disk to Writer -func (cs *FileCarStore) writeShardBlocks(ctx context.Context, sh *CarShard, w io.Writer) error { +func (cs *FileCarStore) writeShardBlocks(ctx context.Context, sh *CarShard, shardOut io.Writer) error { ctx, span := otel.Tracer("carstore").Start(ctx, "writeShardBlocks") defer span.End() @@ -420,7 +420,7 @@ func (cs *FileCarStore) writeShardBlocks(ctx context.Context, sh *CarShard, w io return err } - _, err = io.Copy(w, fi) + _, err = io.Copy(shardOut, fi) if err != nil { return err } diff --git a/carstore/repo_test.go b/carstore/repo_test.go index bdddda424..c05b2ca16 100644 --- a/carstore/repo_test.go +++ b/carstore/repo_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "path/filepath" "testing" @@ -24,7 +25,7 @@ import ( "gorm.io/gorm" ) -func testCarStore() (CarStore, func(), error) { +func testCarStore(t testing.TB) (CarStore, func(), error) { tempdir, err := os.MkdirTemp("", "msttest-") if err != nil { return nil, nil, err @@ -55,8 +56,9 @@ func testCarStore() (CarStore, func(), error) { }, nil } -func testSqliteCarStore() (CarStore, func(), error) { +func testSqliteCarStore(t testing.TB) (CarStore, func(), error) { sqs := &SQLiteStore{} + sqs.log = slogForTest(t) err := sqs.Open(":memory:") if err != nil { return nil, nil, err @@ -64,7 +66,7 @@ func testSqliteCarStore() (CarStore, func(), error) { return sqs, func() {}, nil } -type testFactory func() (CarStore, func(), error) +type testFactory func(t testing.TB) (CarStore, func(), error) var backends = map[string]testFactory{ "cartore": testCarStore, @@ -95,7 +97,7 @@ func TestBasicOperation(ot *testing.T) { for fname, tf := range backends { ot.Run(fname, func(t *testing.T) { - cs, cleanup, err := tf() + cs, cleanup, err := tf(t) if err != nil { t.Fatal(err) } @@ -178,7 +180,7 @@ func TestBasicOperation(ot *testing.T) { func TestRepeatedCompactions(t *testing.T) { ctx := context.TODO() - cs, cleanup, err := testCarStore() + cs, cleanup, err := testCarStore(t) if err != nil { t.Fatal(err) } @@ -339,13 +341,13 @@ func setupRepo(ctx context.Context, bs blockstore.Blockstore, mkprofile bool) (c func BenchmarkRepoWritesCarstore(b *testing.B) { ctx := context.TODO() - cs, cleanup, err := testCarStore() + cs, cleanup, err := testCarStore(b) innerBenchmarkRepoWritesCarstore(b, ctx, cs, cleanup, err) } func BenchmarkRepoWritesSqliteCarstore(b *testing.B) { ctx := context.TODO() - cs, cleanup, err := testSqliteCarStore() + cs, cleanup, err := testSqliteCarStore(b) innerBenchmarkRepoWritesCarstore(b, ctx, cs, cleanup, err) } func innerBenchmarkRepoWritesCarstore(b *testing.B, ctx context.Context, cs CarStore, cleanup func(), err error) { @@ -489,7 +491,7 @@ func TestDuplicateBlockAcrossShards(ot *testing.T) { for fname, tf := range backends { ot.Run(fname, func(t *testing.T) { - cs, cleanup, err := tf() + cs, cleanup, err := tf(t) if err != nil { t.Fatal(err) } @@ -616,3 +618,19 @@ func TestDuplicateBlockAcrossShards(ot *testing.T) { }) } } + +type testWriter struct { + t testing.TB +} + +func (tw testWriter) Write(p []byte) (n int, err error) { + tw.t.Log(string(p)) + return len(p), nil +} + +func slogForTest(t testing.TB) *slog.Logger { + hopts := slog.HandlerOptions{ + Level: slog.LevelDebug, + } + return slog.New(slog.NewTextHandler(&testWriter{t}, &hopts)) +} diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index 73ee9c797..9cf9e6284 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -6,6 +6,9 @@ import ( "database/sql" "errors" "fmt" + "io" + "log/slog" + "github.com/bluesky-social/indigo/models" blockformat "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -17,36 +20,56 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "go.opentelemetry.io/otel" - "io" - "log/slog" ) -//var log = logging.Logger("sqstore") +// var log = logging.Logger("sqstore") type SQLiteStore struct { - db *sql.DB + dbPath string + db *sql.DB + + log *slog.Logger lastShardCache lastShardCache } func (sqs *SQLiteStore) Open(path string) error { + if sqs.log == nil { + sqs.log = slog.Default() + } + sqs.log.Debug("open db", "path", path) db, err := sql.Open("sqlite3", path) if err != nil { return fmt.Errorf("%s: sqlite could not open, %w", path, err) } - _, err = db.Exec("CREATE TABLE IF NOT EXISTS blocks (uid int, cid blob, rev varchar, root varchar, block blob, PRIMARY KEY(uid,cid))") + sqs.db = db + sqs.dbPath = path + err = sqs.createTables() if err != nil { - return fmt.Errorf("%s: create table blocks..., %w", path, err) + return fmt.Errorf("%s: sqlite could not create tables, %w", path, err) } - sqs.db = db sqs.lastShardCache.source = sqs sqs.lastShardCache.Init() return nil } +func (sqs *SQLiteStore) createTables() error { + tx, err := sqs.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + _, err = tx.Exec("CREATE TABLE IF NOT EXISTS blocks (uid int, cid blob, rev varchar, root blob, block blob, PRIMARY KEY(uid,cid));") + if err != nil { + return fmt.Errorf("%s: create table blocks..., %w", sqs.dbPath, err) + } + return tx.Commit() +} + // writeNewShard needed for DeltaSession.CloseWithRoot func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { sqWriteNewShard.Inc() + sqs.log.Debug("write shard", "uid", user, "root", root, "rev", rev, "nblocks", len(blks)) // TODO: trace span here // this is "write many blocks", "write one block" is above in putBlock(). keep them in sync. buf := new(bytes.Buffer) @@ -56,7 +79,7 @@ func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev str } offset := hnw - insertStatement, err := sqs.db.PrepareContext(ctx, "INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?)") + insertStatement, err := sqs.db.PrepareContext(ctx, "INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?) ON CONFLICT (uid,cid) DO UPDATE SET rev=excluded.rev, root=excluded.root, block=excluded.block") if err != nil { return nil, fmt.Errorf("bad block insert sql, %w", err) } @@ -74,10 +97,12 @@ func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev str // TODO: better databases have an insert-many option for a prepared statement dbcid := models.DbCID{CID: bcid} - _, err = insertStatement.ExecContext(ctx, user, dbcid, rev, dbroot, block.RawData()) + blockbytes := block.RawData() + _, err = insertStatement.ExecContext(ctx, user, dbcid, rev, dbroot, blockbytes) if err != nil { return nil, fmt.Errorf("(uid,cid) block store failed, %w", err) } + sqs.log.Debug("put block", "uid", user, "cid", bcid, "size", len(blockbytes)) } shard := CarShard{ @@ -123,12 +148,12 @@ func (sqs *SQLiteStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarS } func (sqs *SQLiteStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { - slog.Warn("TODO: don't call compaction") + sqs.log.Warn("TODO: don't call compaction") return nil, nil } func (sqs *SQLiteStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { - slog.Warn("TODO: don't call compaction targets") + sqs.log.Warn("TODO: don't call compaction targets") return nil, nil } @@ -245,18 +270,67 @@ func (sqs *SQLiteStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) }, nil } +type cartmp struct { + xcid cid.Cid + rev string + root string + block []byte +} + // ReadUserCar // incremental is only ever called true -func (sqs *SQLiteStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error { - //TODO implement me - // TODO: get help understanding what PDS does for this - panic("implement me") +func (sqs *SQLiteStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, shardOut io.Writer) error { + sqGetCar.Inc() + + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return fmt.Errorf("rcar tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT cid,rev,root,block FROM blocks WHERE uid = ? AND rev > ? ORDER BY rev DESC") + if err != nil { + return fmt.Errorf("rcar sql, %w", err) + } + defer qstmt.Close() + rows, err := qstmt.QueryContext(ctx, user, sinceRev) + if err != nil { + return fmt.Errorf("rcar err, %w", err) + } + nblocks := 0 + first := true + for rows.Next() { + var xcid models.DbCID + var xrev string + var xroot models.DbCID + var xblock []byte + err = rows.Scan(&xcid, &xrev, &xroot, &xblock) + if err != nil { + return fmt.Errorf("rcar bad scan, %w", err) + } + if first { + if err := car.WriteHeader(&car.CarHeader{ + Roots: []cid.Cid{xroot.CID}, + Version: 1, + }, shardOut); err != nil { + return fmt.Errorf("rcar bad header, %w", err) + } + first = false + } + nblocks++ + _, err := LdWrite(shardOut, xcid.CID.Bytes(), xblock) + if err != nil { + return fmt.Errorf("rcar bad write, %w", err) + } + } + sqs.log.Debug("read car", "nblocks", nblocks, "since", sinceRev) + sqs.log.Error("TODO is this right?") + return nil } // Stat is only used in a debugging admin handler // don't bother implementing it (for now?) func (sqs *SQLiteStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { - slog.Warn("Stat debugging method not implemented for sqlite store") + sqs.log.Warn("Stat debugging method not implemented for sqlite store") return nil, nil } @@ -272,11 +346,18 @@ func (sqs *SQLiteStore) WipeUserData(ctx context.Context, user models.Uid) error return err } +var txReadOnly = sql.TxOptions{ReadOnly: true} + // HasUidCid needed for NewDeltaSession userView func (sqs *SQLiteStore) HasUidCid(ctx context.Context, user models.Uid, bcid cid.Cid) (bool, error) { // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData sqHas.Inc() - qstmt, err := sqs.db.PrepareContext(ctx, "SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return false, fmt.Errorf("hasUC tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") if err != nil { return false, fmt.Errorf("hasUC sql, %w", err) } @@ -308,14 +389,19 @@ func (sqs *SQLiteStore) Close() error { func (sqs *SQLiteStore) getBlock(ctx context.Context, user models.Uid, bcid cid.Cid) (blockformat.Block, error) { // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData sqGetBlock.Inc() - qstmt, err := sqs.db.PrepareContext(ctx, "SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") - defer qstmt.Close() + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return nil, fmt.Errorf("getb tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") if err != nil { - return nil, fmt.Errorf("hasUC sql, %w", err) + return nil, fmt.Errorf("getb sql, %w", err) } + defer qstmt.Close() rows, err := qstmt.QueryContext(ctx, user, models.DbCID{bcid}) if err != nil { - return nil, fmt.Errorf("hasUC err, %w", err) + return nil, fmt.Errorf("getb err, %w", err) } if rows.Next() { //var rev string @@ -323,7 +409,7 @@ func (sqs *SQLiteStore) getBlock(ctx context.Context, user models.Uid, bcid cid. var blockb []byte err = rows.Scan(&blockb) if err != nil { - return nil, fmt.Errorf("hasUC bad scan, %w", err) + return nil, fmt.Errorf("getb bad scan, %w", err) } return blocks.NewBlock(blockb), nil } @@ -335,18 +421,18 @@ func (sqs *SQLiteStore) getBlockSize(ctx context.Context, user models.Uid, bcid sqGetBlockSize.Inc() qstmt, err := sqs.db.PrepareContext(ctx, "SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") if err != nil { - return 0, fmt.Errorf("hasUC sql, %w", err) + return 0, fmt.Errorf("getbs sql, %w", err) } defer qstmt.Close() rows, err := qstmt.QueryContext(ctx, user, models.DbCID{bcid}) if err != nil { - return 0, fmt.Errorf("hasUC err, %w", err) + return 0, fmt.Errorf("getbs err, %w", err) } if rows.Next() { var out int64 err = rows.Scan(&out) if err != nil { - return 0, fmt.Errorf("hasUC bad scan, %w", err) + return 0, fmt.Errorf("getbs bad scan, %w", err) } return out, nil } @@ -392,6 +478,11 @@ var sqGetBlockSize = promauto.NewCounter(prometheus.CounterOpts{ Help: "get block size sqlite backend", }) +var sqGetCar = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_get_car", + Help: "get block sqlite backend", +}) + var sqHas = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_sq_has", Help: "check block presence sqlite backend", diff --git a/repo/repo.go b/repo/repo.go index acdafcce6..f4e683f4c 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -80,7 +80,7 @@ func IngestRepo(ctx context.Context, bs blockstore.Blockstore, r io.Reader) (cid br, err := car.NewBlockReader(r) if err != nil { - return cid.Undef, err + return cid.Undef, fmt.Errorf("IngestRepo:NewBlockReader: %w", err) } for { @@ -89,11 +89,11 @@ func IngestRepo(ctx context.Context, bs blockstore.Blockstore, r io.Reader) (cid if err == io.EOF { break } - return cid.Undef, err + return cid.Undef, fmt.Errorf("IngestRepo:Next: %w", err) } if err := bs.Put(ctx, blk); err != nil { - return cid.Undef, err + return cid.Undef, fmt.Errorf("IngestRepo:Put: %w", err) } } @@ -104,7 +104,7 @@ func ReadRepoFromCar(ctx context.Context, r io.Reader) (*Repo, error) { bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) root, err := IngestRepo(ctx, bs, r) if err != nil { - return nil, err + return nil, fmt.Errorf("ReadRepoFromCar:IngestRepo: %w", err) } return OpenRepo(ctx, bs, root) From 29cb514d4e9c629363775edded35eae142923faa Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 1 Nov 2024 17:00:06 -0400 Subject: [PATCH 05/30] more sqlite txn --- carstore/sqlite_store.go | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index 9cf9e6284..db23af029 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -79,7 +79,12 @@ func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev str } offset := hnw - insertStatement, err := sqs.db.PrepareContext(ctx, "INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?) ON CONFLICT (uid,cid) DO UPDATE SET rev=excluded.rev, root=excluded.root, block=excluded.block") + tx, err := sqs.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("bad block insert tx, %w", err) + } + defer tx.Rollback() + insertStatement, err := tx.PrepareContext(ctx, "INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?) ON CONFLICT (uid,cid) DO UPDATE SET rev=excluded.rev, root=excluded.root, block=excluded.block") if err != nil { return nil, fmt.Errorf("bad block insert sql, %w", err) } @@ -104,6 +109,10 @@ func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev str } sqs.log.Debug("put block", "uid", user, "cid", bcid, "size", len(blockbytes)) } + err = tx.Commit() + if err != nil { + return nil, fmt.Errorf("bad block insert commit, %w", err) + } shard := CarShard{ Root: models.DbCID{CID: root}, @@ -124,7 +133,12 @@ var ErrNothingThere = errors.New("nothing to read)") // What we actually seem to need from this: last {Rev, Root.CID} func (sqs *SQLiteStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarShard, error) { sqGetLastShard.Inc() - qstmt, err := sqs.db.PrepareContext(ctx, "SELECT rev, root FROM blocks WHERE uid = ? ORDER BY rev DESC LIMIT 1") + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return nil, fmt.Errorf("bad last shard tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT rev, root FROM blocks WHERE uid = ? ORDER BY rev DESC LIMIT 1") if err != nil { return nil, fmt.Errorf("bad last shard sql, %w", err) } @@ -335,7 +349,12 @@ func (sqs *SQLiteStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, e } func (sqs *SQLiteStore) WipeUserData(ctx context.Context, user models.Uid) error { - deleteResult, err := sqs.db.ExecContext(ctx, "DELETE FROM blocks WHERE uid = ?", user) + tx, err := sqs.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("wipe tx, %w", err) + } + defer tx.Rollback() + deleteResult, err := tx.ExecContext(ctx, "DELETE FROM blocks WHERE uid = ?", user) nrows, ierr := deleteResult.RowsAffected() if ierr == nil { sqRowsDeleted.Add(float64(nrows)) @@ -343,6 +362,9 @@ func (sqs *SQLiteStore) WipeUserData(ctx context.Context, user models.Uid) error if err == nil { err = ierr } + if err == nil { + err = tx.Commit() + } return err } @@ -419,7 +441,12 @@ func (sqs *SQLiteStore) getBlock(ctx context.Context, user models.Uid, bcid cid. func (sqs *SQLiteStore) getBlockSize(ctx context.Context, user models.Uid, bcid cid.Cid) (int64, error) { // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData sqGetBlockSize.Inc() - qstmt, err := sqs.db.PrepareContext(ctx, "SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return 0, fmt.Errorf("getbs tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") if err != nil { return 0, fmt.Errorf("getbs sql, %w", err) } From 0b6612a65bf63539ceaace4aaa7ae25e956d98d5 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 1 Nov 2024 20:04:31 -0400 Subject: [PATCH 06/30] index (uid,rev DESC); add readme --- carstore/README.md | 26 ++++++++++++++++++++++++++ carstore/sqlite_store.go | 4 ++++ 2 files changed, 30 insertions(+) create mode 100644 carstore/README.md diff --git a/carstore/README.md b/carstore/README.md new file mode 100644 index 000000000..8b2ce0335 --- /dev/null +++ b/carstore/README.md @@ -0,0 +1,26 @@ +# Carstore + +Store a zillion users of PDS-like repo, with more limited operations (mainly: firehose in, firehose out) + +## Sqlite3 store + +Experimental/demo. + +```sql +CREATE TABLE IF NOT EXISTS blocks (uid int, cid blob, rev varchar, root blob, block blob, PRIMARY KEY(uid,cid)) +CREATE INDEX IF NOT EXISTS blocx_by_rev ON blocks (uid, rev DESC) + +INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?) ON CONFLICT (uid,cid) DO UPDATE SET rev=excluded.rev, root=excluded.root, block=excluded.block + +SELECT rev, root FROM blocks WHERE uid = ? ORDER BY rev DESC LIMIT 1 + +SELECT cid,rev,root,block FROM blocks WHERE uid = ? AND rev > ? ORDER BY rev DESC + +DELETE FROM blocks WHERE uid = ? + +SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1 + +SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1 + +SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1 +``` diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index db23af029..07a59c545 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -63,6 +63,10 @@ func (sqs *SQLiteStore) createTables() error { if err != nil { return fmt.Errorf("%s: create table blocks..., %w", sqs.dbPath, err) } + _, err = tx.Exec("CREATE INDEX IF NOT EXISTS blocx_by_rev ON blocks (uid, rev DESC)") + if err != nil { + return fmt.Errorf("%s: create blocks by rev index, %w", sqs.dbPath, err) + } return tx.Commit() } From 9bc8bf7fde8cd8583a93781a4159d11abdd306f7 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Sun, 3 Nov 2024 00:31:06 -0400 Subject: [PATCH 07/30] add some trace spans --- carstore/sqlite_store.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index 07a59c545..9d1119b4a 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -74,7 +74,8 @@ func (sqs *SQLiteStore) createTables() error { func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { sqWriteNewShard.Inc() sqs.log.Debug("write shard", "uid", user, "root", root, "rev", rev, "nblocks", len(blks)) - // TODO: trace span here + ctx, span := otel.Tracer("carstore").Start(ctx, "writeNewShard") + defer span.End() // this is "write many blocks", "write one block" is above in putBlock(). keep them in sync. buf := new(bytes.Buffer) hnw, err := WriteCarHeader(buf, root) @@ -299,6 +300,8 @@ type cartmp struct { // incremental is only ever called true func (sqs *SQLiteStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, shardOut io.Writer) error { sqGetCar.Inc() + ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") + defer span.End() tx, err := sqs.db.BeginTx(ctx, &txReadOnly) if err != nil { @@ -353,6 +356,8 @@ func (sqs *SQLiteStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, e } func (sqs *SQLiteStore) WipeUserData(ctx context.Context, user models.Uid) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "WipeUserData") + defer span.End() tx, err := sqs.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("wipe tx, %w", err) From 8bf9dcd772e4be1fd56628fc8d637a24d36622f3 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Sun, 3 Nov 2024 01:12:47 -0400 Subject: [PATCH 08/30] flag to enable experimental sqlite carstore --- carstore/sqlite_store.go | 11 +++++++++++ cmd/bigsky/main.go | 14 +++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index 9d1119b4a..897af0ca9 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "log/slog" + "path/filepath" "github.com/bluesky-social/indigo/models" blockformat "github.com/ipfs/go-block-format" @@ -33,6 +34,16 @@ type SQLiteStore struct { lastShardCache lastShardCache } +func NewSqliteStore(csdir string) (*SQLiteStore, error) { + dbpath := filepath.Join(csdir, "db.sqlite3") + out := new(SQLiteStore) + err := out.Open(dbpath) + if err != nil { + return nil, err + } + return out, nil +} + func (sqs *SQLiteStore) Open(path string) error { if sqs.log == nil { sqs.log = slog.Default() diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index c4d7d7b28..5f40ac38e 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -194,6 +194,11 @@ func run(args []string) error { EnvVars: []string{"RELAY_EVENT_PLAYBACK_TTL"}, Value: 72 * time.Hour, }, + &cli.BoolFlag{ + Name: "ex-sqlite-carstore", + Usage: "enable experimental sqlite carstore", + Value: false, + }, } app.Action = runBigsky @@ -307,7 +312,14 @@ func runBigsky(cctx *cli.Context) error { } os.MkdirAll(filepath.Dir(csdir), os.ModePerm) - cstore, err := carstore.NewCarStore(csdb, csdir) + + var cstore carstore.CarStore + if cctx.Bool("ex-sqlite-carstore") { + cstore, err = carstore.NewSqliteStore(csdir) + } else { + // make standard FileCarStore + cstore, err = carstore.NewCarStore(csdb, csdir) + } if err != nil { return err } From e5f84030c9d39f0a88ecba5a91c68a15cb52c325 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Sun, 3 Nov 2024 01:23:58 -0400 Subject: [PATCH 09/30] mkdirs --- carstore/sqlite_store.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index 897af0ca9..a7f45c14c 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "log/slog" + "os" "path/filepath" "github.com/bluesky-social/indigo/models" @@ -34,7 +35,24 @@ type SQLiteStore struct { lastShardCache lastShardCache } +func ensureDir(path string) error { + fi, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return os.MkdirAll(path, 0755) + } + return err + } + if fi.IsDir() { + return nil + } + return fmt.Errorf("%s exists but is not a directory", path) +} + func NewSqliteStore(csdir string) (*SQLiteStore, error) { + if err := ensureDir(csdir); err != nil { + return nil, err + } dbpath := filepath.Join(csdir, "db.sqlite3") out := new(SQLiteStore) err := out.Open(dbpath) From a631350ed8ada999738816626f92baaaec412c5e Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Sun, 3 Nov 2024 01:45:23 -0400 Subject: [PATCH 10/30] nil fix --- carstore/sqlite_store.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index a7f45c14c..7822485dc 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -224,6 +224,9 @@ func (sqs *SQLiteStore) GetUserRepoRev(ctx context.Context, user models.Uid) (st if err != nil { return "", err } + if lastShard == nil { + return "", nil + } if lastShard.ID == 0 { return "", nil } From e8a57d8da2fdf56526f19461aa1f84128985876e Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Sun, 3 Nov 2024 01:46:53 -0400 Subject: [PATCH 11/30] nil fix --- carstore/sqlite_store.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index 7822485dc..93f6ae14f 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -211,6 +211,9 @@ func (sqs *SQLiteStore) GetUserRepoHead(ctx context.Context, user models.Uid) (c if err != nil { return cid.Undef, err } + if lastShard == nil { + return "", nil + } if lastShard.ID == 0 { return cid.Undef, nil } From da9022aa0b7a339f7db7ee36595b90efca73dbd7 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Sun, 3 Nov 2024 01:47:26 -0400 Subject: [PATCH 12/30] nil fix --- carstore/sqlite_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index 93f6ae14f..aaea587ef 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -212,7 +212,7 @@ func (sqs *SQLiteStore) GetUserRepoHead(ctx context.Context, user models.Uid) (c return cid.Undef, err } if lastShard == nil { - return "", nil + return cid.Undef, nil } if lastShard.ID == 0 { return cid.Undef, nil From 06e76d556d4b10e41b853be24398243904b21100 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Thu, 7 Nov 2024 12:33:47 -0500 Subject: [PATCH 13/30] note blocks written in otel span --- carstore/sqlite_store.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index aaea587ef..c1a3310a4 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -6,6 +6,7 @@ import ( "database/sql" "errors" "fmt" + "go.opentelemetry.io/otel/attribute" "io" "log/slog" "os" @@ -126,6 +127,8 @@ func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev str dbroot := models.DbCID{root} + span.SetAttributes(attribute.Int("blocks", len(blks))) + for bcid, block := range blks { // build shard for output firehose nw, err := LdWrite(buf, bcid.Bytes(), block.RawData()) From 1e283b7f8b74a8cc17b6826a5f39a07c425afbfb Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Thu, 7 Nov 2024 20:16:04 -0500 Subject: [PATCH 14/30] wip: scylla backend compiles --- carstore/scylla.go | 530 +++++++++++++++++++++++++++++++++++++++ carstore/sqlite_store.go | 17 +- go.mod | 8 +- go.sum | 15 ++ 4 files changed, 563 insertions(+), 7 deletions(-) create mode 100644 carstore/scylla.go diff --git a/carstore/scylla.go b/carstore/scylla.go new file mode 100644 index 000000000..a6dd12ddf --- /dev/null +++ b/carstore/scylla.go @@ -0,0 +1,530 @@ +package carstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/bluesky-social/indigo/models" + "github.com/gocql/gocql" + blockformat "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipld/go-car" + _ "github.com/mattn/go-sqlite3" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "io" + "log/slog" + "math" + "math/rand/v2" + "time" +) + +// var log = logging.Logger("sqstore") + +type ScyllaStore struct { + WriteSession *gocql.Session + ReadSession *gocql.Session + + // scylla servers + scyllaAddrs []string + // scylla namespace where we find our table + keyspace string + + log *slog.Logger + + lastShardCache lastShardCache +} + +func NewScyllaStore(addrs []string, keyspace string) (*ScyllaStore, error) { + out := new(ScyllaStore) + out.scyllaAddrs = addrs + out.keyspace = keyspace + return out, nil +} + +func (sqs *ScyllaStore) Open(path string) error { + if sqs.log == nil { + sqs.log = slog.Default() + } + sqs.log.Debug("scylla connect", "addrs", sqs.scyllaAddrs) + var err error + + // + // Write session + // + var writeSession *gocql.Session + for retry := 0; ; retry++ { + writeCluster := gocql.NewCluster(sqs.scyllaAddrs...) + writeCluster.Keyspace = sqs.keyspace + // Default port, the client should automatically upgrade to shard-aware port + writeCluster.Port = 9042 + writeCluster.Consistency = gocql.Quorum + writeCluster.RetryPolicy = &ExponentialBackoffRetryPolicy{NumRetries: 10, Min: 100 * time.Millisecond, Max: 10 * time.Second} + writeCluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(gocql.RoundRobinHostPolicy()) + writeSession, err = writeCluster.CreateSession() + if err != nil { + if retry > 200 { + return fmt.Errorf("failed to connect read session too many times: %w", err) + } + sqs.log.Error("failed to connect to ScyllaDB Read Session, retrying in 1s", "retry", retry, "err", err) + time.Sleep(delayForAttempt(retry)) + continue + } + break + } + + // + // Read session + // + var readSession *gocql.Session + for retry := 0; ; retry++ { + readCluster := gocql.NewCluster(sqs.scyllaAddrs...) + readCluster.Keyspace = sqs.keyspace + // Default port, the client should automatically upgrade to shard-aware port + readCluster.Port = 9042 + readCluster.RetryPolicy = &ExponentialBackoffRetryPolicy{NumRetries: 5, Min: 10 * time.Millisecond, Max: 1 * time.Second} + readCluster.Consistency = gocql.One + readCluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(gocql.RoundRobinHostPolicy()) + readSession, err = readCluster.CreateSession() + if err != nil { + if retry > 200 { + return fmt.Errorf("failed to connect read session too many times: %w", err) + } + sqs.log.Error("failed to connect to ScyllaDB Read Session, retrying in 1s", "retry", retry, "err", err) + time.Sleep(delayForAttempt(retry)) + continue + } + break + } + + sqs.WriteSession = writeSession + sqs.ReadSession = readSession + + err = sqs.createTables() + if err != nil { + return fmt.Errorf("%s: scylla could not create tables, %w", path, err) + } + sqs.lastShardCache.source = sqs + sqs.lastShardCache.Init() + return nil +} + +var createTableTexts = []string{ + `CREATE TABLE IF NOT EXISTS blocks (uid bigint, cid blob, rev varchar, root blob, block blob, PRIMARY KEY((uid,cid)))`, + `CREATE INDEX IF NOT EXISTS block_by_rev ON blocks (uid, rev)`, +} + +func (sqs *ScyllaStore) createTables() error { + for i, text := range createTableTexts { + err := sqs.WriteSession.Query(text).Exec() + if err != nil { + return fmt.Errorf("scylla create table statement [%d] %v: %w", i, text, err) + } + } + return nil +} + +// writeNewShard needed for DeltaSession.CloseWithRoot +func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { + scWriteNewShard.Inc() + sqs.log.Debug("write shard", "uid", user, "root", root, "rev", rev, "nblocks", len(blks)) + ctx, span := otel.Tracer("carstore").Start(ctx, "writeNewShard") + defer span.End() + // this is "write many blocks", "write one block" is above in putBlock(). keep them in sync. + buf := new(bytes.Buffer) + hnw, err := WriteCarHeader(buf, root) + if err != nil { + return nil, fmt.Errorf("failed to write car header: %w", err) + } + offset := hnw + + dbroot := models.DbCID{CID: root} + + span.SetAttributes(attribute.Int("blocks", len(blks))) + + for bcid, block := range blks { + // build shard for output firehose + nw, err := LdWrite(buf, bcid.Bytes(), block.RawData()) + if err != nil { + return nil, fmt.Errorf("failed to write block: %w", err) + } + offset += nw + + // TODO: better databases have an insert-many option for a prepared statement - BUT scylla BATCH doesn't apply if the batch crosses partition keys + dbcid := models.DbCID{CID: bcid} + blockbytes := block.RawData() + // TODO: how good is the cql auto-prepare interning? + err = sqs.WriteSession.Query( + `INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?)`, + user, dbcid, rev, dbroot, blockbytes, + ).Idempotent(true).Exec() + if err != nil { + return nil, fmt.Errorf("(uid,cid) block store failed, %w", err) + } + sqs.log.Debug("put block", "uid", user, "cid", bcid, "size", len(blockbytes)) + } + + shard := CarShard{ + Root: models.DbCID{CID: root}, + DataStart: hnw, + Seq: seq, + Usr: user, + Rev: rev, + } + + sqs.lastShardCache.put(&shard) + + return buf.Bytes(), nil +} + +// GetLastShard nedeed for NewDeltaSession indirectly through lastShardCache +// What we actually seem to need from this: last {Rev, Root.CID} +func (sqs *ScyllaStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarShard, error) { + scGetLastShard.Inc() + var rev string + var rootb models.DbCID + err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks WHERE uid = ? ORDER BY rev DESC LIMIT 1`, uid).Scan(&rev, &rootb) + if err != nil { + return nil, fmt.Errorf("last shard err, %w", err) + } + return &CarShard{ + Root: rootb, + Rev: rev, + }, nil +} + +func (sqs *ScyllaStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { + sqs.log.Warn("TODO: don't call compaction") + return nil, nil +} + +func (sqs *ScyllaStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { + sqs.log.Warn("TODO: don't call compaction targets") + return nil, nil +} + +func (sqs *ScyllaStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { + // TODO: same as FileCarStore; re-unify + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return cid.Undef, err + } + if lastShard == nil { + return cid.Undef, nil + } + if lastShard.ID == 0 { + return cid.Undef, nil + } + + return lastShard.Root.CID, nil +} + +func (sqs *ScyllaStore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { + // TODO: same as FileCarStore; re-unify + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return "", err + } + if lastShard == nil { + return "", nil + } + if lastShard.ID == 0 { + return "", nil + } + + return lastShard.Rev, nil +} + +func (sqs *ScyllaStore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { + // TODO: same as FileCarStore, re-unify + ctx, span := otel.Tracer("carstore").Start(ctx, "ImportSlice") + defer span.End() + + carr, err := car.NewCarReader(bytes.NewReader(carslice)) + if err != nil { + return cid.Undef, nil, err + } + + if len(carr.Header.Roots) != 1 { + return cid.Undef, nil, fmt.Errorf("invalid car file, header must have a single root (has %d)", len(carr.Header.Roots)) + } + + ds, err := sqs.NewDeltaSession(ctx, uid, since) + if err != nil { + return cid.Undef, nil, fmt.Errorf("new delta session failed: %w", err) + } + + var cids []cid.Cid + for { + blk, err := carr.Next() + if err != nil { + if err == io.EOF { + break + } + return cid.Undef, nil, err + } + + cids = append(cids, blk.Cid()) + + if err := ds.Put(ctx, blk); err != nil { + return cid.Undef, nil, err + } + } + + return carr.Header.Roots[0], ds, nil +} + +func (sqs *ScyllaStore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession") + defer span.End() + + // TODO: ensure that we don't write updates on top of the wrong head + // this needs to be a compare and swap type operation + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return nil, fmt.Errorf("NewDeltaSession, lsc, %w", err) + } + + if lastShard == nil { + lastShard = &zeroShard + } + + if since != nil && *since != lastShard.Rev { + return nil, fmt.Errorf("revision mismatch: %s != %s: %w", *since, lastShard.Rev, ErrRepoBaseMismatch) + } + + return &DeltaSession{ + fresh: blockstore.NewBlockstore(datastore.NewMapDatastore()), + blks: make(map[cid.Cid]blockformat.Block), + base: &sqliteUserView{ + uid: user, + sqs: sqs, + }, + user: user, + baseCid: lastShard.Root.CID, + cs: sqs, + seq: lastShard.Seq + 1, + lastRev: lastShard.Rev, + }, nil +} + +func (sqs *ScyllaStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { + return &DeltaSession{ + base: &sqliteUserView{ + uid: user, + sqs: sqs, + }, + readonly: true, + user: user, + cs: sqs, + }, nil +} + +// ReadUserCar +// incremental is only ever called true +func (sqs *ScyllaStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, shardOut io.Writer) error { + scGetCar.Inc() + ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") + defer span.End() + + rows := sqs.ReadSession.Query(`SELECT cid,rev,root,block FROM blocks WHERE uid = ? AND rev > ? ORDER BY rev DESC`, user, sinceRev).Iter() + nblocks := 0 + first := true + for { + var xcid models.DbCID + var xrev string + var xroot models.DbCID + var xblock []byte + ok := rows.Scan(&xcid, &xrev, &xroot, &xblock) + if !ok { + break + } + if first { + if err := car.WriteHeader(&car.CarHeader{ + Roots: []cid.Cid{xroot.CID}, + Version: 1, + }, shardOut); err != nil { + rows.Close() + return fmt.Errorf("rcar bad header, %w", err) + } + first = false + } + nblocks++ + _, err := LdWrite(shardOut, xcid.CID.Bytes(), xblock) + if err != nil { + rows.Close() + return fmt.Errorf("rcar bad write, %w", err) + } + } + err := rows.Close() + if err != nil { + return fmt.Errorf("rcar bad read, %w", err) + } + sqs.log.Debug("read car", "nblocks", nblocks, "since", sinceRev) + return nil +} + +// Stat is only used in a debugging admin handler +// don't bother implementing it (for now?) +func (sqs *ScyllaStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { + sqs.log.Warn("Stat debugging method not implemented for sqlite store") + return nil, nil +} + +func (sqs *ScyllaStore) WipeUserData(ctx context.Context, user models.Uid) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "WipeUserData") + defer span.End() + err := sqs.WriteSession.Query("DELETE FROM blocks WHERE uid = ?", user).Exec() + return err +} + +// HasUidCid needed for NewDeltaSession userView +func (sqs *ScyllaStore) HasUidCid(ctx context.Context, user models.Uid, bcid cid.Cid) (bool, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + scHas.Inc() + var rev string + var rootb models.DbCID + err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1`, user, models.DbCID{CID: bcid}).Scan(&rev, rootb) + if err != nil { + return false, fmt.Errorf("hasUC bad scan, %w", err) + } + return true, nil +} + +func (sqs *ScyllaStore) CarStore() CarStore { + return sqs +} + +func (sqs *ScyllaStore) Close() error { + sqs.WriteSession.Close() + sqs.ReadSession.Close() + return nil +} + +func (sqs *ScyllaStore) getBlock(ctx context.Context, user models.Uid, bcid cid.Cid) (blockformat.Block, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + scGetBlock.Inc() + var blockb []byte + err := sqs.ReadSession.Query("SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, models.DbCID{CID: bcid}).Scan(&blockb) + if err != nil { + return nil, fmt.Errorf("getb err, %w", err) + } + return blocks.NewBlock(blockb), nil +} + +func (sqs *ScyllaStore) getBlockSize(ctx context.Context, user models.Uid, bcid cid.Cid) (int64, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + scGetBlockSize.Inc() + var out int64 + err := sqs.ReadSession.Query("SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, models.DbCID{CID: bcid}).Scan(&out) + if err != nil { + return 0, fmt.Errorf("getbs err, %w", err) + } + return out, nil +} + +var scRowsDeleted = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_rows_deleted", + Help: "User rows deleted in sclite backend", +}) + +var scGetBlock = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_get_block", + Help: "get block sclite backend", +}) + +var scGetBlockSize = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_get_block_size", + Help: "get block size sclite backend", +}) + +var scGetCar = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_get_car", + Help: "get block sclite backend", +}) + +var scHas = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_has", + Help: "check block presence sclite backend", +}) + +var scGetLastShard = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_get_last_shard", + Help: "get last shard sclite backend", +}) + +var scWriteNewShard = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_write_shard", + Help: "write shard blocks sclite backend", +}) + +// TODO: copied from tango, re-unify? +// ExponentialBackoffRetryPolicy sleeps between attempts +type ExponentialBackoffRetryPolicy struct { + NumRetries int + Min, Max time.Duration +} + +func (e *ExponentialBackoffRetryPolicy) napTime(attempts int) time.Duration { + return getExponentialTime(e.Min, e.Max, attempts) +} + +func (e *ExponentialBackoffRetryPolicy) Attempt(q gocql.RetryableQuery) bool { + if q.Attempts() > e.NumRetries { + return false + } + time.Sleep(e.napTime(q.Attempts())) + return true +} + +// used to calculate exponentially growing time +func getExponentialTime(min time.Duration, max time.Duration, attempts int) time.Duration { + if min <= 0 { + min = 100 * time.Millisecond + } + if max <= 0 { + max = 10 * time.Second + } + minFloat := float64(min) + napDuration := minFloat * math.Pow(2, float64(attempts-1)) + // add some jitter + napDuration += rand.Float64()*minFloat - (minFloat / 2) + if napDuration > float64(max) { + return time.Duration(max) + } + return time.Duration(napDuration) +} + +// GetRetryType returns the retry type for the given error +func (e *ExponentialBackoffRetryPolicy) GetRetryType(err error) gocql.RetryType { + // Retry timeouts and/or contention errors on the same host + if errors.Is(err, gocql.ErrTimeoutNoResponse) || + errors.Is(err, gocql.ErrNoStreams) || + errors.Is(err, gocql.ErrTooManyTimeouts) { + return gocql.Retry + } + + // Retry next host on unavailable errors + if errors.Is(err, gocql.ErrUnavailable) || + errors.Is(err, gocql.ErrConnectionClosed) || + errors.Is(err, gocql.ErrSessionClosed) { + return gocql.RetryNextHost + } + + // Otherwise don't retry + return gocql.Rethrow +} + +func delayForAttempt(attempt int) time.Duration { + if attempt < 50 { + return time.Millisecond * 5 + } + + return time.Second +} diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index c1a3310a4..761f87b60 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -125,7 +125,7 @@ func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev str } defer insertStatement.Close() - dbroot := models.DbCID{root} + dbroot := models.DbCID{CID: root} span.SetAttributes(attribute.Int("blocks", len(blks))) @@ -382,7 +382,6 @@ func (sqs *SQLiteStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR } } sqs.log.Debug("read car", "nblocks", nblocks, "since", sinceRev) - sqs.log.Error("TODO is this right?") return nil } @@ -431,7 +430,7 @@ func (sqs *SQLiteStore) HasUidCid(ctx context.Context, user models.Uid, bcid cid return false, fmt.Errorf("hasUC sql, %w", err) } defer qstmt.Close() - rows, err := qstmt.QueryContext(ctx, user, models.DbCID{bcid}) + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{CID: bcid}) if err != nil { return false, fmt.Errorf("hasUC err, %w", err) } @@ -468,7 +467,7 @@ func (sqs *SQLiteStore) getBlock(ctx context.Context, user models.Uid, bcid cid. return nil, fmt.Errorf("getb sql, %w", err) } defer qstmt.Close() - rows, err := qstmt.QueryContext(ctx, user, models.DbCID{bcid}) + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{CID: bcid}) if err != nil { return nil, fmt.Errorf("getb err, %w", err) } @@ -498,7 +497,7 @@ func (sqs *SQLiteStore) getBlockSize(ctx context.Context, user models.Uid, bcid return 0, fmt.Errorf("getbs sql, %w", err) } defer qstmt.Close() - rows, err := qstmt.QueryContext(ctx, user, models.DbCID{bcid}) + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{CID: bcid}) if err != nil { return 0, fmt.Errorf("getbs err, %w", err) } @@ -513,8 +512,14 @@ func (sqs *SQLiteStore) getBlockSize(ctx context.Context, user models.Uid, bcid return 0, nil } +type sqliteUserViewInner interface { + HasUidCid(ctx context.Context, user models.Uid, bcid cid.Cid) (bool, error) + getBlock(ctx context.Context, user models.Uid, bcid cid.Cid) (blockformat.Block, error) + getBlockSize(ctx context.Context, user models.Uid, bcid cid.Cid) (int64, error) +} + type sqliteUserView struct { - sqs *SQLiteStore + sqs sqliteUserViewInner uid models.Uid } diff --git a/go.mod b/go.mod index 4b942517b..f1fd88aa7 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/flosch/pongo2/v6 v6.0.0 github.com/go-redis/cache/v9 v9.0.0 github.com/goccy/go-json v0.10.2 + github.com/gocql/gocql v0.0.0-00010101000000-000000000000 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/gorilla/websocket v1.5.1 github.com/hashicorp/go-retryablehttp v0.7.5 @@ -79,6 +80,8 @@ require ( require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-redis/redis v6.15.9+incompatible // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/klauspost/compress v1.17.3 // indirect @@ -91,6 +94,7 @@ require ( github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + gopkg.in/inf.v0 v0.9.1 // indirect ) require ( @@ -137,7 +141,7 @@ require ( github.com/lestrrat-go/option v1.0.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/mattn/go-sqlite3 v1.14.22 github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect @@ -173,3 +177,5 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) + +replace github.com/gocql/gocql => github.com/scylladb/gocql v1.14.4 diff --git a/go.sum b/go.sum index dfc251448..0bff6925e 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/brianvoe/gofakeit/v6 v6.25.0 h1:ZpFjktOpLZUeF8q223o0rUuXtA+m5qW5srjvVi+JkXk= github.com/brianvoe/gofakeit/v6 v6.25.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -189,6 +193,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -231,6 +237,8 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= @@ -563,6 +571,8 @@ github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/samber/slog-echo v1.8.0 h1:DQQRtAliSvQw+ScEdu5gv3jbHu9cCTzvHuTD8GDv7zI= github.com/samber/slog-echo v1.8.0/go.mod h1:0ab2AwcciQXNAXEcjkHwD9okOh9vEHEYn8xP97ocuhM= +github.com/scylladb/gocql v1.14.4 h1:MhevwCfyAraQ6RvZYFO3pF4Lt0YhvQlfg8Eo2HEqVQA= +github.com/scylladb/gocql v1.14.4/go.mod h1:ZLEJ0EVE5JhmtxIW2stgHq/v1P4fWap0qyyXSKyV8K0= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= @@ -777,6 +787,7 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= @@ -1061,6 +1072,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1095,3 +1108,5 @@ lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From c5e910776b733d205b659a50231bf1b650923b11 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 8 Nov 2024 10:49:53 -0500 Subject: [PATCH 15/30] connect scylla to main --- carstore/scylla.go | 8 ++++++-- cmd/bigsky/main.go | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/carstore/scylla.go b/carstore/scylla.go index a6dd12ddf..250a546ea 100644 --- a/carstore/scylla.go +++ b/carstore/scylla.go @@ -45,10 +45,14 @@ func NewScyllaStore(addrs []string, keyspace string) (*ScyllaStore, error) { out := new(ScyllaStore) out.scyllaAddrs = addrs out.keyspace = keyspace + err := out.Open() + if err != nil { + return nil, err + } return out, nil } -func (sqs *ScyllaStore) Open(path string) error { +func (sqs *ScyllaStore) Open() error { if sqs.log == nil { sqs.log = slog.Default() } @@ -108,7 +112,7 @@ func (sqs *ScyllaStore) Open(path string) error { err = sqs.createTables() if err != nil { - return fmt.Errorf("%s: scylla could not create tables, %w", path, err) + return fmt.Errorf("scylla could not create tables, %w", err) } sqs.lastShardCache.source = sqs sqs.lastShardCache.Init() diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index 5f40ac38e..ad0857f16 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -199,6 +199,11 @@ func run(args []string) error { Usage: "enable experimental sqlite carstore", Value: false, }, + &cli.StringSliceFlag{ + Name: "ex-scylla-carstore", + Usage: "scylla server addresses for storage backend, probably comma separated, urfave/cli is unclear", + Value: &cli.StringSlice{}, + }, } app.Action = runBigsky @@ -314,7 +319,10 @@ func runBigsky(cctx *cli.Context) error { os.MkdirAll(filepath.Dir(csdir), os.ModePerm) var cstore carstore.CarStore - if cctx.Bool("ex-sqlite-carstore") { + scyllaAddrs := cctx.StringSlice("ex-scylla-carstore") + if len(scyllaAddrs) != 0 { + cstore, err = carstore.NewScyllaStore(scyllaAddrs, "cs") + } else if cctx.Bool("ex-sqlite-carstore") { cstore, err = carstore.NewSqliteStore(csdir) } else { // make standard FileCarStore From d0e0a273c4832c46f8f6ed4695249502cf7f416b Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 8 Nov 2024 11:15:16 -0500 Subject: [PATCH 16/30] allow posting pds crawl with limits --- bgs/admin.go | 41 ++++++++++++++++++++++++++++++++++------- bgs/fedmgr.go | 9 ++++++++- bgs/handlers.go | 2 +- testing/utils.go | 14 ++++++++------ 4 files changed, 51 insertions(+), 15 deletions(-) diff --git a/bgs/admin.go b/bgs/admin.go index b2f385cd5..c167db40f 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -353,13 +353,35 @@ func (bgs *BGS) handleAdminUnbanDomain(c echo.Context) error { }) } +type PDSRates struct { + PerSecond int64 `json:"per_second,omitempty"` + PerHour int64 `json:"per_hour,omitempty"` + PerDay int64 `json:"per_day,omitempty"` + CrawlRate int64 `json:"crawl_rate,omitempty"` + RepoLimit int64 `json:"repo_limit,omitempty"` +} + +func (pr *PDSRates) FromSlurper(s *Slurper) { + if pr.PerSecond == 0 { + pr.PerHour = s.DefaultPerSecondLimit + } + if pr.PerHour == 0 { + pr.PerHour = s.DefaultPerHourLimit + } + if pr.PerDay == 0 { + pr.PerDay = s.DefaultPerDayLimit + } + if pr.CrawlRate == 0 { + pr.CrawlRate = int64(s.DefaultCrawlLimit) + } + if pr.RepoLimit == 0 { + pr.RepoLimit = s.DefaultRepoLimit + } +} + type RateLimitChangeRequest struct { - Host string `json:"host"` - PerSecond int64 `json:"per_second"` - PerHour int64 `json:"per_hour"` - PerDay int64 `json:"per_day"` - CrawlRate int64 `json:"crawl_rate"` - RepoLimit int64 `json:"repo_limit"` + Host string `json:"host"` + PDSRates } func (bgs *BGS) handleAdminChangePDSRateLimits(e echo.Context) error { @@ -592,6 +614,9 @@ func (bgs *BGS) handleAdminAddTrustedDomain(e echo.Context) error { type AdminRequestCrawlRequest struct { Hostname string `json:"hostname"` + + // optional: + PDSRates } func (bgs *BGS) handleAdminRequestCrawl(e echo.Context) error { @@ -644,6 +669,8 @@ func (bgs *BGS) handleAdminRequestCrawl(e echo.Context) error { } // Skip checking if the server is online for now + rateOverrides := body.PDSRates + rateOverrides.FromSlurper(bgs.slurper) - return bgs.slurper.SubscribeToPds(ctx, host, true, true) // Override Trusted Domain Check + return bgs.slurper.SubscribeToPds(ctx, host, true, true, &rateOverrides) // Override Trusted Domain Check } diff --git a/bgs/fedmgr.go b/bgs/fedmgr.go index 42ce7407c..d6b40ec7a 100644 --- a/bgs/fedmgr.go +++ b/bgs/fedmgr.go @@ -360,7 +360,7 @@ func (s *Slurper) canSlurpHost(host string) bool { return !s.newSubsDisabled } -func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, adminOverride bool) error { +func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, adminOverride bool, rateOverrides *PDSRates) error { // TODO: for performance, lock on the hostname instead of global s.lk.Lock() defer s.lk.Unlock() @@ -394,6 +394,13 @@ func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, adm CrawlRateLimit: float64(s.DefaultCrawlLimit), RepoLimit: s.DefaultRepoLimit, } + if rateOverrides != nil { + npds.RateLimit = float64(rateOverrides.PerSecond) + npds.HourlyEventLimit = rateOverrides.PerHour + npds.DailyEventLimit = rateOverrides.PerDay + npds.CrawlRateLimit = float64(rateOverrides.CrawlRate) + npds.RepoLimit = rateOverrides.RepoLimit + } if err := s.db.Create(&npds).Error; err != nil { return err } diff --git a/bgs/handlers.go b/bgs/handlers.go index da87c9521..b7dd53b49 100644 --- a/bgs/handlers.go +++ b/bgs/handlers.go @@ -185,7 +185,7 @@ func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context, body *comatp // Maybe we could do something with this response later _ = desc - return s.slurper.SubscribeToPds(ctx, host, true, false) + return s.slurper.SubscribeToPds(ctx, host, true, false, nil) } func (s *BGS) handleComAtprotoSyncNotifyOfUpdate(ctx context.Context, body *comatprototypes.SyncNotifyOfUpdate_Input) error { diff --git a/testing/utils.go b/testing/utils.go index 9b076ef17..b89989269 100644 --- a/testing/utils.go +++ b/testing/utils.go @@ -210,12 +210,14 @@ func (tp *TestPDS) BumpLimits(t *testing.T, b *TestRelay) { } limReqBody := bgs.RateLimitChangeRequest{ - Host: u.Host, - PerSecond: 5_000, - PerHour: 100_000, - PerDay: 1_000_000, - RepoLimit: 500_000, - CrawlRate: 50_000, + Host: u.Host, + PDSRates: bgs.PDSRates{ + PerSecond: 5_000, + PerHour: 100_000, + PerDay: 1_000_000, + RepoLimit: 500_000, + CrawlRate: 50_000, + }, } // JSON encode the request body From 190b304235cb2234cb8e666bbc9665d6c55e9095 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 8 Nov 2024 15:44:20 -0500 Subject: [PATCH 17/30] scylla appeasment, index -> materialized view --- carstore/scylla.go | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/carstore/scylla.go b/carstore/scylla.go index 250a546ea..558ce7ad5 100644 --- a/carstore/scylla.go +++ b/carstore/scylla.go @@ -121,7 +121,12 @@ func (sqs *ScyllaStore) Open() error { var createTableTexts = []string{ `CREATE TABLE IF NOT EXISTS blocks (uid bigint, cid blob, rev varchar, root blob, block blob, PRIMARY KEY((uid,cid)))`, - `CREATE INDEX IF NOT EXISTS block_by_rev ON blocks (uid, rev)`, + //`CREATE INDEX IF NOT EXISTS block_by_rev ON blocks (uid, rev)`, + `CREATE MATERIALIZED VIEW IF NOT EXISTS blocks_by_uidrev +AS SELECT uid, rev, cid, root +FROM blocks +WHERE uid IS NOT NULL AND rev IS NOT NULL AND cid IS NOT NULL +PRIMARY KEY ((uid), rev, cid) WITH CLUSTERING ORDER BY (rev DESC)`, } func (sqs *ScyllaStore) createTables() error { @@ -193,7 +198,7 @@ func (sqs *ScyllaStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarS scGetLastShard.Inc() var rev string var rootb models.DbCID - err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks WHERE uid = ? ORDER BY rev DESC LIMIT 1`, uid).Scan(&rev, &rootb) + err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks_by_uidrev WHERE uid = ? ORDER BY rev DESC LIMIT 1`, uid).Scan(&rev, &rootb) if err != nil { return nil, fmt.Errorf("last shard err, %w", err) } @@ -337,39 +342,45 @@ func (sqs *ScyllaStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") defer span.End() - rows := sqs.ReadSession.Query(`SELECT cid,rev,root,block FROM blocks WHERE uid = ? AND rev > ? ORDER BY rev DESC`, user, sinceRev).Iter() + cidchan := make(chan models.DbCID, 100) + + go func() { + defer close(cidchan) + cids := sqs.ReadSession.Query(`SELECT cid FROM blocks_by_uidrev WHERE uid = ? AND rev > ? ORDER BY rev DESC`, user, sinceRev).Iter() + for { + var xcid models.DbCID + ok := cids.Scan(&xcid) + if !ok { + break + } + cidchan <- xcid + } + }() nblocks := 0 first := true - for { - var xcid models.DbCID + for xcid := range cidchan { var xrev string var xroot models.DbCID var xblock []byte - ok := rows.Scan(&xcid, &xrev, &xroot, &xblock) - if !ok { - break + err := sqs.ReadSession.Query("SELECT rev, root, block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, xcid).Scan(&xrev, &xroot, &xblock) + if err != nil { + return fmt.Errorf("rcar bad read, %w", err) } if first { if err := car.WriteHeader(&car.CarHeader{ Roots: []cid.Cid{xroot.CID}, Version: 1, }, shardOut); err != nil { - rows.Close() return fmt.Errorf("rcar bad header, %w", err) } first = false } nblocks++ - _, err := LdWrite(shardOut, xcid.CID.Bytes(), xblock) + _, err = LdWrite(shardOut, xcid.CID.Bytes(), xblock) if err != nil { - rows.Close() return fmt.Errorf("rcar bad write, %w", err) } } - err := rows.Close() - if err != nil { - return fmt.Errorf("rcar bad read, %w", err) - } sqs.log.Debug("read car", "nblocks", nblocks, "since", sinceRev) return nil } @@ -385,6 +396,7 @@ func (sqs *ScyllaStore) WipeUserData(ctx context.Context, user models.Uid) error ctx, span := otel.Tracer("carstore").Start(ctx, "WipeUserData") defer span.End() err := sqs.WriteSession.Query("DELETE FROM blocks WHERE uid = ?", user).Exec() + scUsersWiped.Inc() return err } @@ -433,8 +445,8 @@ func (sqs *ScyllaStore) getBlockSize(ctx context.Context, user models.Uid, bcid return out, nil } -var scRowsDeleted = promauto.NewCounter(prometheus.CounterOpts{ - Name: "bgs_sc_rows_deleted", +var scUsersWiped = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_users_wiped", Help: "User rows deleted in sclite backend", }) From 367255ddbea4f711ce64843ce23ab32cb1377b68 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Mon, 11 Nov 2024 10:02:19 -0500 Subject: [PATCH 18/30] not found is nil return, not err --- carstore/scylla.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/carstore/scylla.go b/carstore/scylla.go index 558ce7ad5..0e5d2d311 100644 --- a/carstore/scylla.go +++ b/carstore/scylla.go @@ -199,6 +199,9 @@ func (sqs *ScyllaStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarS var rev string var rootb models.DbCID err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks_by_uidrev WHERE uid = ? ORDER BY rev DESC LIMIT 1`, uid).Scan(&rev, &rootb) + if errors.Is(err, gocql.ErrNotFound) { + return nil, nil + } if err != nil { return nil, fmt.Errorf("last shard err, %w", err) } From 29679536fa7293fd0b0bc54e9a11072691abd7bd Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Mon, 11 Nov 2024 12:37:12 -0500 Subject: [PATCH 19/30] cql marshal --- models/dbcid.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/models/dbcid.go b/models/dbcid.go index 366a0e829..d64ae0bc6 100644 --- a/models/dbcid.go +++ b/models/dbcid.go @@ -4,6 +4,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "github.com/gocql/gocql" "github.com/ipfs/go-cid" ) @@ -62,3 +63,15 @@ func (dbc *DbCID) UnmarshalJSON(b []byte) error { func (dbc *DbCID) GormDataType() string { return "bytes" } + +func (dbc *DbCID) MarshalCQL(info gocql.TypeInfo) ([]byte, error) { + return dbc.CID.Bytes(), nil +} +func (dbc *DbCID) UnmarshalCQL(info gocql.TypeInfo, data []byte) error { + xcid, err := cid.Cast(data) + if err != nil { + return err + } + dbc.CID = xcid + return nil +} From 43a68b2ec00fe4d758bf4d6982643155708e4a43 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Mon, 11 Nov 2024 12:53:45 -0500 Subject: [PATCH 20/30] explicit Cid <-> []byte for scylla --- carstore/scylla.go | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/carstore/scylla.go b/carstore/scylla.go index 0e5d2d311..8a6e9ffe2 100644 --- a/carstore/scylla.go +++ b/carstore/scylla.go @@ -153,7 +153,7 @@ func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev str } offset := hnw - dbroot := models.DbCID{CID: root} + dbroot := root.Bytes() span.SetAttributes(attribute.Int("blocks", len(blks))) @@ -166,7 +166,7 @@ func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev str offset += nw // TODO: better databases have an insert-many option for a prepared statement - BUT scylla BATCH doesn't apply if the batch crosses partition keys - dbcid := models.DbCID{CID: bcid} + dbcid := bcid.Bytes() blockbytes := block.RawData() // TODO: how good is the cql auto-prepare interning? err = sqs.WriteSession.Query( @@ -197,7 +197,7 @@ func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev str func (sqs *ScyllaStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarShard, error) { scGetLastShard.Inc() var rev string - var rootb models.DbCID + var rootb []byte err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks_by_uidrev WHERE uid = ? ORDER BY rev DESC LIMIT 1`, uid).Scan(&rev, &rootb) if errors.Is(err, gocql.ErrNotFound) { return nil, nil @@ -205,8 +205,12 @@ func (sqs *ScyllaStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarS if err != nil { return nil, fmt.Errorf("last shard err, %w", err) } + xcid, cidErr := cid.Cast(rootb) + if cidErr != nil { + return nil, fmt.Errorf("last shard bad cid, %w", cidErr) + } return &CarShard{ - Root: rootb, + Root: models.DbCID{CID: xcid}, Rev: rev, }, nil } @@ -345,17 +349,23 @@ func (sqs *ScyllaStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") defer span.End() - cidchan := make(chan models.DbCID, 100) + cidchan := make(chan cid.Cid, 100) go func() { defer close(cidchan) cids := sqs.ReadSession.Query(`SELECT cid FROM blocks_by_uidrev WHERE uid = ? AND rev > ? ORDER BY rev DESC`, user, sinceRev).Iter() + defer cids.Close() for { - var xcid models.DbCID - ok := cids.Scan(&xcid) + var cidb []byte + ok := cids.Scan(&cidb) if !ok { break } + xcid, cidErr := cid.Cast(cidb) + if cidErr != nil { + sqs.log.Warn("ReadUserCar bad cid", "err", cidErr) + continue + } cidchan <- xcid } }() @@ -363,15 +373,19 @@ func (sqs *ScyllaStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR first := true for xcid := range cidchan { var xrev string - var xroot models.DbCID + var xroot []byte var xblock []byte - err := sqs.ReadSession.Query("SELECT rev, root, block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, xcid).Scan(&xrev, &xroot, &xblock) + err := sqs.ReadSession.Query("SELECT rev, root, block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, xcid.Bytes()).Scan(&xrev, &xroot, &xblock) if err != nil { return fmt.Errorf("rcar bad read, %w", err) } if first { + rootCid, cidErr := cid.Cast(xroot) + if cidErr != nil { + return fmt.Errorf("rcar bad rootcid, %w", err) + } if err := car.WriteHeader(&car.CarHeader{ - Roots: []cid.Cid{xroot.CID}, + Roots: []cid.Cid{rootCid}, Version: 1, }, shardOut); err != nil { return fmt.Errorf("rcar bad header, %w", err) @@ -379,7 +393,7 @@ func (sqs *ScyllaStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR first = false } nblocks++ - _, err = LdWrite(shardOut, xcid.CID.Bytes(), xblock) + _, err = LdWrite(shardOut, xcid.Bytes(), xblock) if err != nil { return fmt.Errorf("rcar bad write, %w", err) } @@ -408,8 +422,8 @@ func (sqs *ScyllaStore) HasUidCid(ctx context.Context, user models.Uid, bcid cid // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData scHas.Inc() var rev string - var rootb models.DbCID - err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1`, user, models.DbCID{CID: bcid}).Scan(&rev, rootb) + var rootb []byte + err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1`, user, bcid.Bytes()).Scan(&rev, &rootb) if err != nil { return false, fmt.Errorf("hasUC bad scan, %w", err) } @@ -430,7 +444,7 @@ func (sqs *ScyllaStore) getBlock(ctx context.Context, user models.Uid, bcid cid. // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData scGetBlock.Inc() var blockb []byte - err := sqs.ReadSession.Query("SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, models.DbCID{CID: bcid}).Scan(&blockb) + err := sqs.ReadSession.Query("SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, bcid.Bytes()).Scan(&blockb) if err != nil { return nil, fmt.Errorf("getb err, %w", err) } @@ -441,7 +455,7 @@ func (sqs *ScyllaStore) getBlockSize(ctx context.Context, user models.Uid, bcid // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData scGetBlockSize.Inc() var out int64 - err := sqs.ReadSession.Query("SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, models.DbCID{CID: bcid}).Scan(&out) + err := sqs.ReadSession.Query("SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, bcid.Bytes()).Scan(&out) if err != nil { return 0, fmt.Errorf("getbs err, %w", err) } From 27741bca3cba2d8fc5f8ccfe134906702242c22e Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Mon, 11 Nov 2024 13:07:39 -0500 Subject: [PATCH 21/30] use secondary index to delet blocks by uid --- carstore/scylla.go | 60 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/carstore/scylla.go b/carstore/scylla.go index 8a6e9ffe2..ef480f8f9 100644 --- a/carstore/scylla.go +++ b/carstore/scylla.go @@ -412,9 +412,46 @@ func (sqs *ScyllaStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, e func (sqs *ScyllaStore) WipeUserData(ctx context.Context, user models.Uid) error { ctx, span := otel.Tracer("carstore").Start(ctx, "WipeUserData") defer span.End() - err := sqs.WriteSession.Query("DELETE FROM blocks WHERE uid = ?", user).Exec() + + // LOL, can't do this if primary key is (uid,cid) because that's hashed with no scan! + //err := sqs.WriteSession.Query("DELETE FROM blocks WHERE uid = ?", user).Exec() + + cidchan := make(chan cid.Cid, 100) + + go func() { + defer close(cidchan) + cids := sqs.ReadSession.Query(`SELECT cid FROM blocks_by_uidrev WHERE uid = ?`, user).Iter() + defer cids.Close() + for { + var cidb []byte + ok := cids.Scan(&cidb) + if !ok { + break + } + xcid, cidErr := cid.Cast(cidb) + if cidErr != nil { + sqs.log.Warn("ReadUserCar bad cid", "err", cidErr) + continue + } + cidchan <- xcid + } + }() + nblocks := 0 + errcount := 0 + for xcid := range cidchan { + err := sqs.ReadSession.Query("DELETE FROM blocks WHERE uid = ? AND cid = ?", user, xcid.Bytes()).Exec() + if err != nil { + sqs.log.Warn("ReadUserCar bad delete, %w", err) + errcount++ + if errcount > 10 { + return err + } + } + nblocks++ + } scUsersWiped.Inc() - return err + scBlocksDeleted.Add(float64(nblocks)) + return nil } // HasUidCid needed for NewDeltaSession userView @@ -464,37 +501,42 @@ func (sqs *ScyllaStore) getBlockSize(ctx context.Context, user models.Uid, bcid var scUsersWiped = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_sc_users_wiped", - Help: "User rows deleted in sclite backend", + Help: "User rows deleted in scylla backend", +}) + +var scBlocksDeleted = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_blocks_deleted", + Help: "User blocks deleted in scylla backend", }) var scGetBlock = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_sc_get_block", - Help: "get block sclite backend", + Help: "get block scylla backend", }) var scGetBlockSize = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_sc_get_block_size", - Help: "get block size sclite backend", + Help: "get block size scylla backend", }) var scGetCar = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_sc_get_car", - Help: "get block sclite backend", + Help: "get block scylla backend", }) var scHas = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_sc_has", - Help: "check block presence sclite backend", + Help: "check block presence scylla backend", }) var scGetLastShard = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_sc_get_last_shard", - Help: "get last shard sclite backend", + Help: "get last shard scylla backend", }) var scWriteNewShard = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_sc_write_shard", - Help: "write shard blocks sclite backend", + Help: "write shard blocks scylla backend", }) // TODO: copied from tango, re-unify? From e48e72f2111b6bc60343e32a907e01d343b63dd3 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Mon, 11 Nov 2024 16:46:40 -0500 Subject: [PATCH 22/30] add time histograms on major scylla queries --- carstore/scylla.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/carstore/scylla.go b/carstore/scylla.go index ef480f8f9..da2344971 100644 --- a/carstore/scylla.go +++ b/carstore/scylla.go @@ -143,6 +143,7 @@ func (sqs *ScyllaStore) createTables() error { func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { scWriteNewShard.Inc() sqs.log.Debug("write shard", "uid", user, "root", root, "rev", rev, "nblocks", len(blks)) + start := time.Now() ctx, span := otel.Tracer("carstore").Start(ctx, "writeNewShard") defer span.End() // this is "write many blocks", "write one block" is above in putBlock(). keep them in sync. @@ -189,6 +190,8 @@ func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev str sqs.lastShardCache.put(&shard) + dt := time.Since(start).Seconds() + scWriteTimes.Observe(dt) return buf.Bytes(), nil } @@ -348,6 +351,7 @@ func (sqs *ScyllaStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR scGetCar.Inc() ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") defer span.End() + start := time.Now() cidchan := make(chan cid.Cid, 100) @@ -398,7 +402,9 @@ func (sqs *ScyllaStore) ReadUserCar(ctx context.Context, user models.Uid, sinceR return fmt.Errorf("rcar bad write, %w", err) } } + span.SetAttributes(attribute.Int("blocks", nblocks)) sqs.log.Debug("read car", "nblocks", nblocks, "since", sinceRev) + scReadCarTimes.Observe(time.Since(start).Seconds()) return nil } @@ -480,11 +486,14 @@ func (sqs *ScyllaStore) Close() error { func (sqs *ScyllaStore) getBlock(ctx context.Context, user models.Uid, bcid cid.Cid) (blockformat.Block, error) { // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData scGetBlock.Inc() + start := time.Now() var blockb []byte err := sqs.ReadSession.Query("SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, bcid.Bytes()).Scan(&blockb) if err != nil { return nil, fmt.Errorf("getb err, %w", err) } + dt := time.Since(start) + scGetTimes.Observe(dt.Seconds()) return blocks.NewBlock(blockb), nil } @@ -539,6 +548,33 @@ var scWriteNewShard = promauto.NewCounter(prometheus.CounterOpts{ Help: "write shard blocks scylla backend", }) +var timeBuckets []float64 +var scWriteTimes prometheus.Histogram +var scGetTimes prometheus.Histogram +var scReadCarTimes prometheus.Histogram + +func init() { + timeBuckets = make([]float64, 0, 20) + timeBuckets[0] = 0.000_0100 + i := 0 + for timeBuckets[i] < 1 && len(timeBuckets) < 20 { + timeBuckets = append(timeBuckets, timeBuckets[i]*2) + i++ + } + scWriteTimes = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "bgs_sc_write_times", + Buckets: timeBuckets, + }) + scGetTimes = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "bgs_sc_get_times", + Buckets: timeBuckets, + }) + scReadCarTimes = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "bgs_sc_readcar_times", + Buckets: timeBuckets, + }) +} + // TODO: copied from tango, re-unify? // ExponentialBackoffRetryPolicy sleeps between attempts type ExponentialBackoffRetryPolicy struct { From b20aac61135185cd69893d60d6db66869c74d7e9 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Tue, 12 Nov 2024 21:02:21 -0500 Subject: [PATCH 23/30] fix --- carstore/scylla.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carstore/scylla.go b/carstore/scylla.go index da2344971..d9adbb4aa 100644 --- a/carstore/scylla.go +++ b/carstore/scylla.go @@ -554,7 +554,7 @@ var scGetTimes prometheus.Histogram var scReadCarTimes prometheus.Histogram func init() { - timeBuckets = make([]float64, 0, 20) + timeBuckets = make([]float64, 1, 20) timeBuckets[0] = 0.000_0100 i := 0 for timeBuckets[i] < 1 && len(timeBuckets) < 20 { From 2fe848e495fc744d8d4c44e074e20b43dba325d2 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Wed, 13 Nov 2024 06:17:19 -0500 Subject: [PATCH 24/30] more info on bad repo compare --- cmd/gosky/debug.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gosky/debug.go b/cmd/gosky/debug.go index 75d231cfc..547131aeb 100644 --- a/cmd/gosky/debug.go +++ b/cmd/gosky/debug.go @@ -882,7 +882,7 @@ var debugCompareReposCmd = &cli.Command{ rep1, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo1bytes)) if err != nil { - logger.Fatalf("reading repo: %s", err) + logger.Fatalf("reading repo (got %d bytes): %s", len(repo1bytes), err) return } }() @@ -899,7 +899,7 @@ var debugCompareReposCmd = &cli.Command{ rep2, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo2bytes)) if err != nil { - logger.Fatalf("reading repo: %s", err) + logger.Fatalf("reading repo (god %d bytes): %s", len(repo2bytes), err) return } }() From 6afe8e46f78d994c057c955023320b94f24ca0c6 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Wed, 13 Nov 2024 12:44:11 -0500 Subject: [PATCH 25/30] cleanup --- carstore/README.md | 19 +++++++++++++++++-- carstore/bs.go | 21 ++++++++++----------- carstore/scylla.go | 13 ++++--------- carstore/sqlite_store.go | 6 ++---- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/carstore/README.md b/carstore/README.md index 8b2ce0335..90880defb 100644 --- a/carstore/README.md +++ b/carstore/README.md @@ -1,10 +1,25 @@ # Carstore -Store a zillion users of PDS-like repo, with more limited operations (mainly: firehose in, firehose out) +Store a zillion users of PDS-like repo, with more limited operations (mainly: firehose in, firehose out). -## Sqlite3 store +## [ScyllaStore](scylla.go) + +Blocks stored in ScyllaDB. +User and PDS metadata stored in gorm (PostgreSQL or sqlite3). + +## [FileCarStore](bs.go) + +Store 'car slices' from PDS source subscribeRepo firehose streams to filesystem. +Store metadata to gorm postgresql (or sqlite3). +Periodic compaction of car slices into fewer larger car slices. +User and PDS metadata stored in gorm (PostgreSQL or sqlite3). +FileCarStore was the first production carstore and used through at least 2024-11. + +## [SQLiteStore](sqlite_store.go) Experimental/demo. +Blocks stored in trivial local sqlite3 schema. +Minimal reference implementation from which fancy scalable/performant implementations may be derived. ```sql CREATE TABLE IF NOT EXISTS blocks (uid int, cid blob, rev varchar, root blob, block blob, PRIMARY KEY(uid,cid)) diff --git a/carstore/bs.go b/carstore/bs.go index 8657d2ded..821d417d7 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -18,7 +18,6 @@ import ( blockformat "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" blockstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" @@ -99,13 +98,16 @@ func NewCarStore(meta *gorm.DB, root string) (CarStore, error) { return out, nil } +// userView needs these things to get into the underlying block store +// implemented by CarStoreGormMeta type userViewSource interface { HasUidCid(ctx context.Context, user models.Uid, k cid.Cid) (bool, error) LookupBlockRef(ctx context.Context, k cid.Cid) (path string, offset int64, user models.Uid, err error) } +// wrapper into a block store that keeps track of which user we are working on behalf of type userView struct { - cs userViewSource // TODO: interface-ify, used for .meta.HasUidCid and .meta.LookupBlockRef + cs userViewSource user models.Uid cache map[cid.Cid]blockformat.Block @@ -279,18 +281,15 @@ type minBlockstore interface { } type DeltaSession struct { - fresh blockstore.Blockstore - blks map[cid.Cid]blockformat.Block - rmcids map[cid.Cid]bool - //base blockstore.Blockstore + blks map[cid.Cid]blockformat.Block + rmcids map[cid.Cid]bool base minBlockstore user models.Uid baseCid cid.Cid seq int readonly bool - // cs *FileCarStore // TODO: this is only needed for CloseWithRoot to write back delta session modifications, interface-ify - cs shardWriter - lastRev string + cs shardWriter + lastRev string } func (cs *FileCarStore) checkLastShardCache(user models.Uid) *CarShard { @@ -327,8 +326,7 @@ func (cs *FileCarStore) NewDeltaSession(ctx context.Context, user models.Uid, si } return &DeltaSession{ - fresh: blockstore.NewBlockstore(datastore.NewMapDatastore()), - blks: make(map[cid.Cid]blockformat.Block), + blks: make(map[cid.Cid]blockformat.Block), base: &userView{ user: user, cs: cs.meta, @@ -591,6 +589,7 @@ func WriteCarHeader(w io.Writer, root cid.Cid) (int64, error) { return hnw, nil } +// shardWriter.writeNewShard called from inside DeltaSession.CloseWithRoot type shardWriter interface { // writeNewShard stores blocks in `blks` arg and creates a new shard to propagate out to our firehose writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) diff --git a/carstore/scylla.go b/carstore/scylla.go index d9adbb4aa..8f872c02a 100644 --- a/carstore/scylla.go +++ b/carstore/scylla.go @@ -9,8 +9,6 @@ import ( "github.com/gocql/gocql" blockformat "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-libipfs/blocks" "github.com/ipld/go-car" _ "github.com/mattn/go-sqlite3" @@ -25,8 +23,6 @@ import ( "time" ) -// var log = logging.Logger("sqstore") - type ScyllaStore struct { WriteSession *gocql.Session ReadSession *gocql.Session @@ -121,6 +117,7 @@ func (sqs *ScyllaStore) Open() error { var createTableTexts = []string{ `CREATE TABLE IF NOT EXISTS blocks (uid bigint, cid blob, rev varchar, root blob, block blob, PRIMARY KEY((uid,cid)))`, + // This is the INDEX I wish we could use, but scylla can't do it so we MATERIALIZED VIEW instead //`CREATE INDEX IF NOT EXISTS block_by_rev ON blocks (uid, rev)`, `CREATE MATERIALIZED VIEW IF NOT EXISTS blocks_by_uidrev AS SELECT uid, rev, cid, root @@ -146,7 +143,6 @@ func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev str start := time.Now() ctx, span := otel.Tracer("carstore").Start(ctx, "writeNewShard") defer span.End() - // this is "write many blocks", "write one block" is above in putBlock(). keep them in sync. buf := new(bytes.Buffer) hnw, err := WriteCarHeader(buf, root) if err != nil { @@ -166,10 +162,10 @@ func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev str } offset += nw - // TODO: better databases have an insert-many option for a prepared statement - BUT scylla BATCH doesn't apply if the batch crosses partition keys + // TODO: scylla BATCH doesn't apply if the batch crosses partition keys; BUT, we may be able to send many blocks concurrently? dbcid := bcid.Bytes() blockbytes := block.RawData() - // TODO: how good is the cql auto-prepare interning? + // we're relying on cql auto-prepare, no 'PreparedStatement' err = sqs.WriteSession.Query( `INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?)`, user, dbcid, rev, dbroot, blockbytes, @@ -319,8 +315,7 @@ func (sqs *ScyllaStore) NewDeltaSession(ctx context.Context, user models.Uid, si } return &DeltaSession{ - fresh: blockstore.NewBlockstore(datastore.NewMapDatastore()), - blks: make(map[cid.Cid]blockformat.Block), + blks: make(map[cid.Cid]blockformat.Block), base: &sqliteUserView{ uid: user, sqs: sqs, diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go index 761f87b60..18a8467db 100644 --- a/carstore/sqlite_store.go +++ b/carstore/sqlite_store.go @@ -15,8 +15,6 @@ import ( "github.com/bluesky-social/indigo/models" blockformat "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-libipfs/blocks" "github.com/ipld/go-car" _ "github.com/mattn/go-sqlite3" @@ -301,8 +299,7 @@ func (sqs *SQLiteStore) NewDeltaSession(ctx context.Context, user models.Uid, si } return &DeltaSession{ - fresh: blockstore.NewBlockstore(datastore.NewMapDatastore()), - blks: make(map[cid.Cid]blockformat.Block), + blks: make(map[cid.Cid]blockformat.Block), base: &sqliteUserView{ uid: user, sqs: sqs, @@ -518,6 +515,7 @@ type sqliteUserViewInner interface { getBlockSize(ctx context.Context, user models.Uid, bcid cid.Cid) (int64, error) } +// TODO: rename, used by both sqlite and scylla type sqliteUserView struct { sqs sqliteUserViewInner uid models.Uid From 2bb6244468cf1c2004324a9075f99599fd43b34d Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 15 Nov 2024 09:46:39 -0500 Subject: [PATCH 26/30] fix arg name --- cmd/bigsky/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index a06592702..23c8853d1 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -206,7 +206,7 @@ func run(args []string) error { Value: false, }, &cli.StringSliceFlag{ - Name: "ex-scylla-carstore", + Name: "scylla-carstore", Usage: "scylla server addresses for storage backend, probably comma separated, urfave/cli is unclear", Value: &cli.StringSlice{}, }, @@ -325,7 +325,7 @@ func runBigsky(cctx *cli.Context) error { os.MkdirAll(filepath.Dir(csdir), os.ModePerm) var cstore carstore.CarStore - scyllaAddrs := cctx.StringSlice("ex-scylla-carstore") + scyllaAddrs := cctx.StringSlice("scylla-carstore") if len(scyllaAddrs) != 0 { cstore, err = carstore.NewScyllaStore(scyllaAddrs, "cs") } else if cctx.Bool("ex-sqlite-carstore") { From 6026f75d8e1049fd350b26132cad1cff9fbd6f83 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Wed, 11 Dec 2024 16:57:32 -0500 Subject: [PATCH 27/30] env RELAY_SCYLLA_NODES setup cleanup --- cmd/bigsky/main.go | 57 ++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index 5ccc8e010..7a91f0922 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -222,9 +222,10 @@ func run(args []string) error { Value: false, }, &cli.StringSliceFlag{ - Name: "scylla-carstore", - Usage: "scylla server addresses for storage backend, probably comma separated, urfave/cli is unclear", - Value: &cli.StringSlice{}, + Name: "scylla-carstore", + Usage: "scylla server addresses for storage backend, comma separated", + Value: &cli.StringSlice{}, + EnvVars: []string{"RELAY_SCYLLA_NODES"}, }, } @@ -321,48 +322,50 @@ func runBigsky(cctx *cli.Context) error { return err } - slog.Info("setting up main database") dburl := cctx.String("db-url") + slog.Info("setting up main database", "url", dburl) db, err := cliutil.SetupDatabase(dburl, cctx.Int("max-metadb-connections")) if err != nil { return err } - - slog.Info("setting up carstore database") - csdburl := cctx.String("carstore-db-url") - csdb, err := cliutil.SetupDatabase(csdburl, cctx.Int("max-carstore-connections")) - if err != nil { - return err - } - if cctx.Bool("db-tracing") { if err := db.Use(tracing.NewPlugin()); err != nil { return err } - if err := csdb.Use(tracing.NewPlugin()); err != nil { - return err - } - } - - csdirs := []string{csdir} - if paramDirs := cctx.StringSlice("carstore-shard-dirs"); len(paramDirs) > 0 { - csdirs = paramDirs - } - - for _, csd := range csdirs { - if err := os.MkdirAll(filepath.Dir(csd), os.ModePerm); err != nil { - return err - } } var cstore carstore.CarStore scyllaAddrs := cctx.StringSlice("scylla-carstore") + sqliteStore := cctx.Bool("ex-sqlite-carstore") if len(scyllaAddrs) != 0 { + slog.Info("starting scylla carstore", "addrs", scyllaAddrs) cstore, err = carstore.NewScyllaStore(scyllaAddrs, "cs") - } else if cctx.Bool("ex-sqlite-carstore") { + } else if sqliteStore { + slog.Info("starting sqlite carstore", "dir", csdir) cstore, err = carstore.NewSqliteStore(csdir) } else { // make standard FileCarStore + csdburl := cctx.String("carstore-db-url") + slog.Info("setting up carstore database", "url", csdburl) + csdb, err := cliutil.SetupDatabase(csdburl, cctx.Int("max-carstore-connections")) + if err != nil { + return err + } + if cctx.Bool("db-tracing") { + if err := csdb.Use(tracing.NewPlugin()); err != nil { + return err + } + } + csdirs := []string{csdir} + if paramDirs := cctx.StringSlice("carstore-shard-dirs"); len(paramDirs) > 0 { + csdirs = paramDirs + } + + for _, csd := range csdirs { + if err := os.MkdirAll(filepath.Dir(csd), os.ModePerm); err != nil { + return err + } + } cstore, err = carstore.NewCarStore(csdb, csdirs) } From 805ae5b6fdcf0cfa77fc6a7fc0fa2c9958df2c44 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Tue, 17 Dec 2024 11:45:29 -0500 Subject: [PATCH 28/30] fix usage of ImportNewRepo --- indexer/repofetch.go | 4 +++- repomgr/repomgr.go | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/indexer/repofetch.go b/indexer/repofetch.go index 8ce68bb5f..1e93612d8 100644 --- a/indexer/repofetch.go +++ b/indexer/repofetch.go @@ -141,8 +141,10 @@ func (rf *RepoFetcher) FetchAndIndexRepo(ctx context.Context, job *crawlWork) er } } + revp := &rev if rev == "" { span.SetAttributes(attribute.Bool("full", true)) + revp = nil } c := models.ClientForPds(&pds) @@ -153,7 +155,7 @@ func (rf *RepoFetcher) FetchAndIndexRepo(ctx context.Context, job *crawlWork) er return err } - if err := rf.repoman.ImportNewRepo(ctx, ai.Uid, ai.Did, bytes.NewReader(repo), &rev); err != nil { + if err := rf.repoman.ImportNewRepo(ctx, ai.Uid, ai.Did, bytes.NewReader(repo), revp); err != nil { span.RecordError(err) if ipld.IsNotFound(err) || errors.Is(err, io.EOF) || errors.Is(err, fs.ErrNotExist) { diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go index d2c3766f3..6c349a1c9 100644 --- a/repomgr/repomgr.go +++ b/repomgr/repomgr.go @@ -912,6 +912,9 @@ func (rm *RepoManager) ImportNewRepo(ctx context.Context, user models.Uid, repoD return err } + if rev != nil && *rev == "" { + rev = nil + } if rev == nil { // if 'rev' is nil, this implies a fresh sync. // in this case, ignore any existing blocks we have and treat this like a clean import. From d709ae9d7b96b12f96358608234c7554ebe97b04 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Wed, 18 Dec 2024 01:37:59 -0500 Subject: [PATCH 29/30] func ptr -> interface --- indexer/crawler.go | 15 +++++++++++---- indexer/indexer.go | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/indexer/crawler.go b/indexer/crawler.go index 526da9bb6..39c709332 100644 --- a/indexer/crawler.go +++ b/indexer/crawler.go @@ -14,10 +14,12 @@ import ( ) type CrawlDispatcher struct { + // from Crawl() ingest chan *models.ActorInfo repoSync chan *crawlWork + // from AddToCatchupQueue() catchup chan *crawlWork complete chan models.Uid @@ -26,7 +28,7 @@ type CrawlDispatcher struct { todo map[models.Uid]*crawlWork inProgress map[models.Uid]*crawlWork - doRepoCrawl func(context.Context, *crawlWork) error + repoFetcher CrawlRepoFetcher concurrency int @@ -35,7 +37,12 @@ type CrawlDispatcher struct { done chan struct{} } -func NewCrawlDispatcher(repoFn func(context.Context, *crawlWork) error, concurrency int, log *slog.Logger) (*CrawlDispatcher, error) { +// this is what we need of RepoFetcher +type CrawlRepoFetcher interface { + FetchAndIndexRepo(ctx context.Context, job *crawlWork) error +} + +func NewCrawlDispatcher(repoFetcher CrawlRepoFetcher, concurrency int, log *slog.Logger) (*CrawlDispatcher, error) { if concurrency < 1 { return nil, fmt.Errorf("must specify a non-zero positive integer for crawl dispatcher concurrency") } @@ -45,7 +52,7 @@ func NewCrawlDispatcher(repoFn func(context.Context, *crawlWork) error, concurre repoSync: make(chan *crawlWork), complete: make(chan models.Uid), catchup: make(chan *crawlWork), - doRepoCrawl: repoFn, + repoFetcher: repoFetcher, concurrency: concurrency, todo: make(map[models.Uid]*crawlWork), inProgress: make(map[models.Uid]*crawlWork), @@ -221,7 +228,7 @@ func (c *CrawlDispatcher) fetchWorker() { for { select { case job := <-c.repoSync: - if err := c.doRepoCrawl(context.TODO(), job); err != nil { + if err := c.repoFetcher.FetchAndIndexRepo(context.TODO(), job); err != nil { c.log.Error("failed to perform repo crawl", "did", job.act.Did, "err", err) } diff --git a/indexer/indexer.go b/indexer/indexer.go index e6a324e9e..6920c7fb6 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -69,7 +69,7 @@ func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events } if crawl { - c, err := NewCrawlDispatcher(fetcher.FetchAndIndexRepo, fetcher.MaxConcurrency, ix.log) + c, err := NewCrawlDispatcher(fetcher, fetcher.MaxConcurrency, ix.log) if err != nil { return nil, err } From 16c36b85798e9e47e416abde045195df3677b6e5 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Wed, 18 Dec 2024 01:51:58 -0500 Subject: [PATCH 30/30] comment --- indexer/crawler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/indexer/crawler.go b/indexer/crawler.go index 39c709332..7e2656dd9 100644 --- a/indexer/crawler.go +++ b/indexer/crawler.go @@ -17,11 +17,12 @@ type CrawlDispatcher struct { // from Crawl() ingest chan *models.ActorInfo - repoSync chan *crawlWork - // from AddToCatchupQueue() catchup chan *crawlWork + // from main loop to fetchWorker() + repoSync chan *crawlWork + complete chan models.Uid maplk sync.Mutex