From b16e12f8ee7c36d4850758b60bfc89405da70391 Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Sun, 12 May 2019 08:33:51 +0100 Subject: [PATCH] withRequestRecords can cause ConcurrentModificationException on the metadata, fixes #721 --- .../stormcrawler/warc/FileTimeSizeRotationPolicy.java | 0 .../stormcrawler/warc/WARCRequestRecordFormat.java | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) mode change 100755 => 100644 external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/FileTimeSizeRotationPolicy.java diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/FileTimeSizeRotationPolicy.java b/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/FileTimeSizeRotationPolicy.java old mode 100755 new mode 100644 diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRequestRecordFormat.java b/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRequestRecordFormat.java index e49e56467..25d5c9ab6 100644 --- a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRequestRecordFormat.java +++ b/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRequestRecordFormat.java @@ -64,8 +64,9 @@ public byte[] format(Tuple tuple) { /* * The request record ID is stored in the metadata so that a WARC * response record can later refer to it. + * Deactivated because of https://github.com/DigitalPebble/storm-crawler/issues/721 */ - metadata.addValue("_request.warc_record_id_", mainID); + // metadata.setValue("_request.warc_record_id_", mainID); int contentLength = httpheaders.length; buffer.append("Content-Length").append(": ")