From 63dafc185fd3f7daf94ceca00aa4eaff35bb5892 Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Wed, 13 Dec 2023 10:54:32 +0000 Subject: [PATCH] Have as many WARCBolt instances as there are workers, fix #64 Signed-off-by: Julien Nioche --- .../java/org/commoncrawl/stormcrawler/news/CrawlTopology.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/commoncrawl/stormcrawler/news/CrawlTopology.java b/src/main/java/org/commoncrawl/stormcrawler/news/CrawlTopology.java index 15f4074..8b7d54c 100644 --- a/src/main/java/org/commoncrawl/stormcrawler/news/CrawlTopology.java +++ b/src/main/java/org/commoncrawl/stormcrawler/news/CrawlTopology.java @@ -94,7 +94,7 @@ protected int run(String[] args) { // take it from feed default output so that the feed files themselves // don't get included - unless we want them too of course! - builder.setBolt("warc", warcbolt).localOrShuffleGrouping("feed"); + builder.setBolt("warc", warcbolt, numWorkers).localOrShuffleGrouping("feed"); BoltDeclarer statusBolt = builder.setBolt("status", new StatusUpdaterBolt(), numWorkers) .localOrShuffleGrouping("fetch", Constants.StatusStreamName)