Added Selenium URL scraper as new datasource; modified column filter to allow detailed matching information

backend/lib/processor.py

@@ -628,7 +628,7 @@ def write_csv_items_and_finish(self, data):
 		self.dataset.update_status("Finished")
 		self.dataset.finish(len(data))
 
-	def write_archive_and_finish(self, files, num_items=None, compression=zipfile.ZIP_STORED):
+	def write_archive_and_finish(self, files, num_items=None, compression=zipfile.ZIP_STORED, finish=True):
 		"""
 		Archive a bunch of files into a zip archive and finish processing
 
@@ -639,6 +639,7 @@ def write_archive_and_finish(self, files, num_items=None, compression=zipfile.ZI
 		  files added to the archive will be used.
 		:param int compression:  Type of compression to use. By default, files
 		  are not compressed, to speed up unarchiving.
+		:param bool finish:  Finish the dataset/job afterwards or not?
 		"""
 		is_folder = False
 		if issubclass(type(files), PurePath):
@@ -665,7 +666,8 @@ def write_archive_and_finish(self, files, num_items=None, compression=zipfile.ZI
 		if num_items is None:
 			num_items = done
 
-		self.dataset.finish(num_items)
+		if finish:
+			self.dataset.finish(num_items)
 
 	def create_standalone(self):
 		"""

backend/lib/search.py

@@ -1,10 +1,13 @@
 import hashlib
+import zipfile
 import secrets
 import shutil
 import random
 import json
 import math
 import csv
+import os
+import copy
 
 from pathlib import Path
 from abc import ABC, abstractmethod
@@ -71,18 +74,19 @@ def process(self):
 				items = self.import_from_file(query_parameters.get("file"))
 			else:
 				items = self.search(query_parameters)
-
 		except WorkerInterruptedException:
 			raise ProcessorInterruptedException("Interrupted while collecting data, trying again later.")
 
 		# Write items to file and update the DataBase status to finished
 		num_items = 0
 		if items:
 			self.dataset.update_status("Writing collected data to dataset file")
-			if results_file.suffix == ".ndjson":
-				num_items = self.items_to_ndjson(items, results_file)
-			elif results_file.suffix == ".csv":
+			if self.extension == "csv":
 				num_items = self.items_to_csv(items, results_file)
+			elif self.extension == "ndjson":
+				num_items = self.items_to_ndjson(items, results_file)
+			elif self.extension == "zip":
+				num_items = self.items_to_archive(items, results_file)
 			else:
 				raise NotImplementedError("Datasource query cannot be saved as %s file" % results_file.suffix)
 
@@ -361,6 +365,22 @@ def items_to_ndjson(self, items, filepath):
 
 		return processed
 
+	def items_to_archive(self, items, filepath):
+		"""
+		Save retrieved items as an archive
+
+		Assumes that items is an iterable with one item, a Path object
+		referring to a folder containing files to be archived. The folder will
+		be removed afterwards.
+
+		:param items:
+		:param filepath:  Where to store the archive
+		:return int:  Number of items
+		"""
+		num_items = len(os.listdir(items))
+		self.write_archive_and_finish(items, None, zipfile.ZIP_STORED, False)
+		return num_items
+
 
 class SearchWithScope(Search, ABC):
 	"""
@@ -404,7 +424,7 @@ def search(self, query):
 			# proportion of items matches
 			# first, get amount of items for all threads in which matching
 			# items occur and that are long enough
-			thread_ids = tuple([post["thread_id"] for post in items])
+			thread_ids = tuple([item["thread_id"] for item in items])
 			self.dataset.update_status("Retrieving thread metadata for %i threads" % len(thread_ids))
 			try:
 				min_length = int(query.get("scope_length", 30))