The process pool executor maybe actually allows parallelization

imagefield/management/commands/process_imagefields.py

@@ -1,5 +1,5 @@
 import sys
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ProcessPoolExecutor
 from fnmatch import fnmatch
 from functools import partial
 
@@ -88,16 +88,19 @@ def _process_field(self, field, options):
             )
 
         fn = partial(
-            self._process_instance,
+            _process_instance,
             field=field,
             housekeep=options.get("housekeep"),
             force=options.get("force"),
         )
 
-        with ThreadPoolExecutor() as executor:
-            for index, instance in enumerate(
+        with ProcessPoolExecutor() as executor:
+            for index, (instance, errors) in enumerate(
                 executor.map(fn, queryset.iterator(chunk_size=100))
             ):
+                if errors:
+                    self.stderr.write("\n".join(errors))
+
                 progress = "*" * (50 * index // count)
                 self.stdout.write(
                     f"\r|{progress.ljust(50)}| {index + 1}/{count}", ending=""
@@ -110,16 +113,18 @@ def _process_field(self, field, options):
 
         self.stdout.write("\r|{}| {}/{}".format("*" * 50, count, count))
 
-    def _process_instance(self, instance, field, housekeep, **kwargs):
-        fieldfile = getattr(instance, field.name)
-        for key in field.formats:
-            try:
-                fieldfile.process(key, **kwargs)
-            except Exception as exc:
-                self.stderr.write(
-                    f"Error while processing {fieldfile.name} ({field.field_label}, #{instance.pk}):\n{exc}\n"
-                )
-                if housekeep == "blank-on-failure":
-                    field.save_form_data(instance, "")
 
-        return instance
+def _process_instance(instance, field, housekeep, **kwargs):
+    fieldfile = getattr(instance, field.name)
+    for key in field.formats:
+        try:
+            fieldfile.process(key, **kwargs)
+        except Exception as exc:
+            if housekeep == "blank-on-failure":
+                field.save_form_data(instance, "")
+
+            return instance, [
+                f"Error while processing {fieldfile.name} ({field.field_label}, #{instance.pk}):\n{exc}\n"
+            ]
+
+    return instance, None