From d709ae9d7b96b12f96358608234c7554ebe97b04 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Wed, 18 Dec 2024 01:37:59 -0500 Subject: [PATCH] func ptr -> interface --- indexer/crawler.go | 15 +++++++++++---- indexer/indexer.go | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/indexer/crawler.go b/indexer/crawler.go index 526da9bb6..39c709332 100644 --- a/indexer/crawler.go +++ b/indexer/crawler.go @@ -14,10 +14,12 @@ import ( ) type CrawlDispatcher struct { + // from Crawl() ingest chan *models.ActorInfo repoSync chan *crawlWork + // from AddToCatchupQueue() catchup chan *crawlWork complete chan models.Uid @@ -26,7 +28,7 @@ type CrawlDispatcher struct { todo map[models.Uid]*crawlWork inProgress map[models.Uid]*crawlWork - doRepoCrawl func(context.Context, *crawlWork) error + repoFetcher CrawlRepoFetcher concurrency int @@ -35,7 +37,12 @@ type CrawlDispatcher struct { done chan struct{} } -func NewCrawlDispatcher(repoFn func(context.Context, *crawlWork) error, concurrency int, log *slog.Logger) (*CrawlDispatcher, error) { +// this is what we need of RepoFetcher +type CrawlRepoFetcher interface { + FetchAndIndexRepo(ctx context.Context, job *crawlWork) error +} + +func NewCrawlDispatcher(repoFetcher CrawlRepoFetcher, concurrency int, log *slog.Logger) (*CrawlDispatcher, error) { if concurrency < 1 { return nil, fmt.Errorf("must specify a non-zero positive integer for crawl dispatcher concurrency") } @@ -45,7 +52,7 @@ func NewCrawlDispatcher(repoFn func(context.Context, *crawlWork) error, concurre repoSync: make(chan *crawlWork), complete: make(chan models.Uid), catchup: make(chan *crawlWork), - doRepoCrawl: repoFn, + repoFetcher: repoFetcher, concurrency: concurrency, todo: make(map[models.Uid]*crawlWork), inProgress: make(map[models.Uid]*crawlWork), @@ -221,7 +228,7 @@ func (c *CrawlDispatcher) fetchWorker() { for { select { case job := <-c.repoSync: - if err := c.doRepoCrawl(context.TODO(), job); err != nil { + if err := c.repoFetcher.FetchAndIndexRepo(context.TODO(), job); err != nil { c.log.Error("failed to perform repo crawl", "did", job.act.Did, "err", err) } diff --git a/indexer/indexer.go b/indexer/indexer.go index e6a324e9e..6920c7fb6 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -69,7 +69,7 @@ func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events } if crawl { - c, err := NewCrawlDispatcher(fetcher.FetchAndIndexRepo, fetcher.MaxConcurrency, ix.log) + c, err := NewCrawlDispatcher(fetcher, fetcher.MaxConcurrency, ix.log) if err != nil { return nil, err }