From 5d0b41cba593141244fd275f08a4f87a42d83066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Sat, 4 Nov 2023 14:40:13 +0100 Subject: [PATCH 1/9] first draft: image dimensions update managemenet command --- filer/management/commands/filer_check.py | 26 ++++++++++++++++++++++++ tests/test_filer_check.py | 8 ++++++++ 2 files changed, 34 insertions(+) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index 3a2de61b3..e522b05cb 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -41,6 +41,13 @@ def add_arguments(self, parser): default=False, help="Delete references in database if files are missing in media folder.", ) + parser.add_argument( + '--image-dimensions', + action='store_true', + dest='image_dimensions', + default=False, + help="Look for images without dimensions set, set them accordingly.", + ) parser.add_argument( '--noinput', '--no-input', @@ -72,6 +79,8 @@ def handle(self, *args, **options): self.stdout.write("Aborted: Delete orphaned files from storage.") return self.verify_storages(options) + if options['image_dimensions']: + self.image_dimensions(options) def verify_references(self, options): from filer.models.filemodels import File @@ -112,3 +121,20 @@ def walk(prefix): filer_public = filer_settings.FILER_STORAGES['public']['main'] storage = import_string(filer_public['ENGINE'])() walk(filer_public['UPLOAD_TO_PREFIX']) + + def image_dimensions(self, options): + from filer.models.filemodels import Image + + no_dimensions = Image.objects.filter( + Q(_width=0) | Q(_width__isnull=True) + ) + self.stdout.write(f"trying to set dimensions on {no_dimensions.count()} files") + for image in no_dimensions: + try: + imgfile = image.file.file + except ValueError: + imgfile = image.file_ptr.file + imgfile.seek(0) + image._width, image._height = VILImage.load(imgfile).size + image.save() + return diff --git a/tests/test_filer_check.py b/tests/test_filer_check.py index 01b951ca0..2890cad42 100644 --- a/tests/test_filer_check.py +++ b/tests/test_filer_check.py @@ -67,3 +67,11 @@ def test_delete_orphans(self): call_command('filer_check', delete_orphans=True, interactive=False, verbosity=0) self.assertFalse(os.path.exists(orphan_file)) + + def test_image_dimensions(self): + self.filer_file._width = 0 + self.filer_file.save() + + call_command('filer_check', image_dimensions=True) + self.filer_file.refresh_from_db() + self.assertFalse(self.filer_file._width == 0) From d25f904c9933d8347f1bf35f65f5b6555d8aebf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Sat, 4 Nov 2023 14:48:18 +0100 Subject: [PATCH 2/9] fix: import corrected --- filer/management/commands/filer_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index e522b05cb..7c1e51b54 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -123,7 +123,7 @@ def walk(prefix): walk(filer_public['UPLOAD_TO_PREFIX']) def image_dimensions(self, options): - from filer.models.filemodels import Image + from filer.models.imagemodels import Image no_dimensions = Image.objects.filter( Q(_width=0) | Q(_width__isnull=True) From 392422021f60a50bf409137ddf0108b0e728c122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Sat, 4 Nov 2023 15:11:21 +0100 Subject: [PATCH 3/9] fix: enable for bitmap images --- filer/management/commands/filer_check.py | 12 +++++++++++- tests/test_filer_check.py | 19 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index 7c1e51b54..a92ed64fa 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -123,7 +123,12 @@ def walk(prefix): walk(filer_public['UPLOAD_TO_PREFIX']) def image_dimensions(self, options): + from tests.utils.custom_image.models import Image from filer.models.imagemodels import Image + from django.db.models import Q + from easy_thumbnails.VIL import Image as VILImage + from filer.utils.compatibility import PILImage + import easy_thumbnails no_dimensions = Image.objects.filter( Q(_width=0) | Q(_width__isnull=True) @@ -135,6 +140,11 @@ def image_dimensions(self, options): except ValueError: imgfile = image.file_ptr.file imgfile.seek(0) - image._width, image._height = VILImage.load(imgfile).size + if image.file.name.endswith('.svg'): + image._width, image._height = VILImage.load(imgfile).size + else: + pil_image = PILImage.open(imgfile) + self._width, self._height = pil_image.size + self._transparent = easy_thumbnails.utils.is_transparent(pil_image) image.save() return diff --git a/tests/test_filer_check.py b/tests/test_filer_check.py index 2890cad42..2140ce0af 100644 --- a/tests/test_filer_check.py +++ b/tests/test_filer_check.py @@ -9,6 +9,7 @@ from filer import settings as filer_settings from filer.models.filemodels import File +from filer.models.imagemodels import Image from tests.helpers import create_image @@ -69,9 +70,19 @@ def test_delete_orphans(self): self.assertFalse(os.path.exists(orphan_file)) def test_image_dimensions(self): - self.filer_file._width = 0 - self.filer_file.save() + + original_filename = 'testimage.jpg' + file_obj = SimpleUploadedFile( + name=original_filename, + content=create_image().tobytes(), + content_type='image/jpeg') + self.filer_image = Image.objects.create( + file=file_obj, + original_filename=original_filename) + + self.filer_image._width = 0 + self.filer_image.save() call_command('filer_check', image_dimensions=True) - self.filer_file.refresh_from_db() - self.assertFalse(self.filer_file._width == 0) + self.filer_image.refresh_from_db() + self.assertNotEqual(self.filer_image._width, 0) From a0998e084b726a911ff293228adfc1210edd6580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Mon, 6 Nov 2023 23:46:49 +0100 Subject: [PATCH 4/9] fix: self>image --- filer/management/commands/filer_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index a92ed64fa..2fc2b5c54 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -144,7 +144,7 @@ def image_dimensions(self, options): image._width, image._height = VILImage.load(imgfile).size else: pil_image = PILImage.open(imgfile) - self._width, self._height = pil_image.size - self._transparent = easy_thumbnails.utils.is_transparent(pil_image) + image._width, image._height = pil_image.size + image._transparent = easy_thumbnails.utils.is_transparent(pil_image) image.save() return From f065b98f055c2bd6f2319899626d919a62ef0767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Tue, 7 Nov 2023 00:21:00 +0100 Subject: [PATCH 5/9] test ok --- filer/management/commands/filer_check.py | 6 +++--- tests/test_filer_check.py | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index 2fc2b5c54..d12322eb7 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -143,8 +143,8 @@ def image_dimensions(self, options): if image.file.name.endswith('.svg'): image._width, image._height = VILImage.load(imgfile).size else: - pil_image = PILImage.open(imgfile) - image._width, image._height = pil_image.size - image._transparent = easy_thumbnails.utils.is_transparent(pil_image) + with PILImage.open(imgfile) as pil_image: + image._width, image._height = pil_image.size + image._transparent = easy_thumbnails.utils.is_transparent(pil_image) image.save() return diff --git a/tests/test_filer_check.py b/tests/test_filer_check.py index 2140ce0af..45fd18adf 100644 --- a/tests/test_filer_check.py +++ b/tests/test_filer_check.py @@ -1,6 +1,7 @@ import os import shutil from io import StringIO +from io import BytesIO from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management import call_command @@ -72,13 +73,16 @@ def test_delete_orphans(self): def test_image_dimensions(self): original_filename = 'testimage.jpg' - file_obj = SimpleUploadedFile( - name=original_filename, - content=create_image().tobytes(), - content_type='image/jpeg') - self.filer_image = Image.objects.create( - file=file_obj, - original_filename=original_filename) + with BytesIO() as jpg: + create_image().save(jpg, format='JPEG') + jpg.seek(0) + file_obj = SimpleUploadedFile( + name=original_filename, + content=jpg.read(), + content_type='image/jpeg') + self.filer_image = Image.objects.create( + file=file_obj, + original_filename=original_filename) self.filer_image._width = 0 self.filer_image.save() From 8b73704627a6503d407d035f0cabbd7c037b5c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Tue, 7 Nov 2023 00:50:08 +0100 Subject: [PATCH 6/9] catch file not fonund and corrupt images --- filer/management/commands/filer_check.py | 21 +++++++++++++------ tests/test_filer_check.py | 26 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index d12322eb7..a8010ca32 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -3,6 +3,7 @@ from django.core.files.storage import DefaultStorage from django.core.management.base import BaseCommand from django.utils.module_loading import import_string +from PIL import UnidentifiedImageError from filer import settings as filer_settings @@ -135,16 +136,24 @@ def image_dimensions(self, options): ) self.stdout.write(f"trying to set dimensions on {no_dimensions.count()} files") for image in no_dimensions: + imgfile = None + if image.file_ptr: + file_holder = image.file_ptr + else: + file_holder = image try: imgfile = image.file.file - except ValueError: - imgfile = image.file_ptr.file - imgfile.seek(0) + imgfile.seek(0) + except (FileNotFoundError): + continue if image.file.name.endswith('.svg'): image._width, image._height = VILImage.load(imgfile).size else: - with PILImage.open(imgfile) as pil_image: - image._width, image._height = pil_image.size - image._transparent = easy_thumbnails.utils.is_transparent(pil_image) + try: + with PILImage.open(imgfile) as pil_image: + image._width, image._height = pil_image.size + image._transparent = easy_thumbnails.utils.is_transparent(pil_image) + except UnidentifiedImageError: + continue image.save() return diff --git a/tests/test_filer_check.py b/tests/test_filer_check.py index 45fd18adf..545db99f2 100644 --- a/tests/test_filer_check.py +++ b/tests/test_filer_check.py @@ -70,6 +70,32 @@ def test_delete_orphans(self): call_command('filer_check', delete_orphans=True, interactive=False, verbosity=0) self.assertFalse(os.path.exists(orphan_file)) + def test_image_dimensions_corrupted_file(self): + original_filename = 'testimage.jpg' + file_obj = SimpleUploadedFile( + name=original_filename, + # corrupted! + content=create_image().tobytes(), + content_type='image/jpeg') + self.filer_image = Image.objects.create( + file=file_obj, + original_filename=original_filename) + + self.filer_image._width = 0 + self.filer_image.save() + + call_command('filer_check', image_dimensions=True) + self.filer_image.refresh_from_db() + self.assertNotEqual(self.filer_image._width, 1) + + def test_image_dimensions_file_not_found(self): + self.filer_image = Image.objects.create( + file="123.jpg", + original_filename="123.jpg") + call_command('filer_check', image_dimensions=True) + self.filer_image.refresh_from_db() + self.assertNotEqual(self.filer_image._width, 1) + def test_image_dimensions(self): original_filename = 'testimage.jpg' From 6aa899527585d6d352bb0c008cf451791e168a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Tue, 7 Nov 2023 00:56:23 +0100 Subject: [PATCH 7/9] fix flake8 and silly error --- filer/management/commands/filer_check.py | 10 ++++++---- tests/test_filer_check.py | 3 +-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index a8010ca32..095a752b2 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -3,6 +3,7 @@ from django.core.files.storage import DefaultStorage from django.core.management.base import BaseCommand from django.utils.module_loading import import_string + from PIL import UnidentifiedImageError from filer import settings as filer_settings @@ -124,12 +125,13 @@ def walk(prefix): walk(filer_public['UPLOAD_TO_PREFIX']) def image_dimensions(self, options): - from tests.utils.custom_image.models import Image - from filer.models.imagemodels import Image from django.db.models import Q + + import easy_thumbnails from easy_thumbnails.VIL import Image as VILImage + + from filer.models.imagemodels import Image from filer.utils.compatibility import PILImage - import easy_thumbnails no_dimensions = Image.objects.filter( Q(_width=0) | Q(_width__isnull=True) @@ -142,7 +144,7 @@ def image_dimensions(self, options): else: file_holder = image try: - imgfile = image.file.file + imgfile = file_holder.file imgfile.seek(0) except (FileNotFoundError): continue diff --git a/tests/test_filer_check.py b/tests/test_filer_check.py index 545db99f2..73a544e32 100644 --- a/tests/test_filer_check.py +++ b/tests/test_filer_check.py @@ -1,7 +1,6 @@ import os import shutil -from io import StringIO -from io import BytesIO +from io import BytesIO, StringIO from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management import call_command From 4df35c72ddb9ed50d431712f7e0ab64f840417f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Tue, 7 Nov 2023 21:02:39 +0100 Subject: [PATCH 8/9] add svg test --- filer/management/commands/filer_check.py | 6 ++- tests/test_filer_check.py | 50 ++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index 095a752b2..7cd8a7ff1 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -148,8 +148,10 @@ def image_dimensions(self, options): imgfile.seek(0) except (FileNotFoundError): continue - if image.file.name.endswith('.svg'): - image._width, image._height = VILImage.load(imgfile).size + if image.file.name.lower().endswith('.svg'): + with VILImage.load(imgfile) as vil_image: + # invalid svg doesnt throw errors + image._width, image._height = vil_image.size else: try: with PILImage.open(imgfile) as pil_image: diff --git a/tests/test_filer_check.py b/tests/test_filer_check.py index 73a544e32..f4577d5d1 100644 --- a/tests/test_filer_check.py +++ b/tests/test_filer_check.py @@ -14,6 +14,16 @@ class FilerCheckTestCase(TestCase): + + svg_file_string = """ + + + + {} + """ + def setUp(self): # ensure that filer_public directory is empty from previous tests storage = import_string(filer_settings.FILER_STORAGES['public']['main']['ENGINE'])() @@ -82,10 +92,7 @@ def test_image_dimensions_corrupted_file(self): self.filer_image._width = 0 self.filer_image.save() - call_command('filer_check', image_dimensions=True) - self.filer_image.refresh_from_db() - self.assertNotEqual(self.filer_image._width, 1) def test_image_dimensions_file_not_found(self): self.filer_image = Image.objects.create( @@ -93,7 +100,7 @@ def test_image_dimensions_file_not_found(self): original_filename="123.jpg") call_command('filer_check', image_dimensions=True) self.filer_image.refresh_from_db() - self.assertNotEqual(self.filer_image._width, 1) + self.assertEqual(self.filer_image._width, 0) def test_image_dimensions(self): @@ -115,3 +122,38 @@ def test_image_dimensions(self): call_command('filer_check', image_dimensions=True) self.filer_image.refresh_from_db() self.assertNotEqual(self.filer_image._width, 0) + + def test_image_dimensions_invalid_svg(self): + + original_filename = 'test.svg' + svg_file = bytes("" + self.svg_file_string, "utf-8") + file_obj = SimpleUploadedFile( + name=original_filename, + content=svg_file, + content_type='image/svg+xml') + self.filer_image = Image.objects.create( + file=file_obj, + original_filename=original_filename) + + self.filer_image._width = 0 + self.filer_image.save() + call_command('filer_check', image_dimensions=True) + + def test_image_dimensions_svg(self): + + original_filename = 'test.svg' + svg_file = bytes(self.svg_file_string, "utf-8") + file_obj = SimpleUploadedFile( + name=original_filename, + content=svg_file, + content_type='image/svg+xml') + self.filer_image = Image.objects.create( + file=file_obj, + original_filename=original_filename) + + self.filer_image._width = 0 + self.filer_image.save() + + call_command('filer_check', image_dimensions=True) + self.filer_image.refresh_from_db() + self.assertNotEqual(self.filer_image._width, 0) From 85cbf62d2e151676f82a07a7cafe321f7bba57f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ben=20St=C3=A4hli?= Date: Tue, 7 Nov 2023 21:09:30 +0100 Subject: [PATCH 9/9] add svg test --- filer/management/commands/filer_check.py | 26 ++++++++++++------------ tests/test_filer_check.py | 5 ++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/filer/management/commands/filer_check.py b/filer/management/commands/filer_check.py index 7cd8a7ff1..3eb77d12b 100644 --- a/filer/management/commands/filer_check.py +++ b/filer/management/commands/filer_check.py @@ -138,7 +138,6 @@ def image_dimensions(self, options): ) self.stdout.write(f"trying to set dimensions on {no_dimensions.count()} files") for image in no_dimensions: - imgfile = None if image.file_ptr: file_holder = image.file_ptr else: @@ -147,17 +146,18 @@ def image_dimensions(self, options): imgfile = file_holder.file imgfile.seek(0) except (FileNotFoundError): - continue - if image.file.name.lower().endswith('.svg'): - with VILImage.load(imgfile) as vil_image: - # invalid svg doesnt throw errors - image._width, image._height = vil_image.size + pass else: - try: - with PILImage.open(imgfile) as pil_image: - image._width, image._height = pil_image.size - image._transparent = easy_thumbnails.utils.is_transparent(pil_image) - except UnidentifiedImageError: - continue - image.save() + if image.file.name.lower().endswith('.svg'): + with VILImage.load(imgfile) as vil_image: + # invalid svg doesnt throw errors + image._width, image._height = vil_image.size + else: + try: + with PILImage.open(imgfile) as pil_image: + image._width, image._height = pil_image.size + image._transparent = easy_thumbnails.utils.is_transparent(pil_image) + except UnidentifiedImageError: + continue + image.save() return diff --git a/tests/test_filer_check.py b/tests/test_filer_check.py index f4577d5d1..f18bd389c 100644 --- a/tests/test_filer_check.py +++ b/tests/test_filer_check.py @@ -100,7 +100,6 @@ def test_image_dimensions_file_not_found(self): original_filename="123.jpg") call_command('filer_check', image_dimensions=True) self.filer_image.refresh_from_db() - self.assertEqual(self.filer_image._width, 0) def test_image_dimensions(self): @@ -121,7 +120,7 @@ def test_image_dimensions(self): call_command('filer_check', image_dimensions=True) self.filer_image.refresh_from_db() - self.assertNotEqual(self.filer_image._width, 0) + self.assertGreater(self.filer_image._width, 0) def test_image_dimensions_invalid_svg(self): @@ -156,4 +155,4 @@ def test_image_dimensions_svg(self): call_command('filer_check', image_dimensions=True) self.filer_image.refresh_from_db() - self.assertNotEqual(self.filer_image._width, 0) + self.assertGreater(self.filer_image._width, 0)