From cca900aa01bdf1894426e9f097deeb5bb3b99ba4 Mon Sep 17 00:00:00 2001 From: Bernardo Heynemann Date: Mon, 24 Jan 2022 18:50:17 -0300 Subject: [PATCH] Smarter thumbor doctor (#1393) * Smarter thumbor doctor Now users can pass their thumbor.conf file in and thumbor-doctor validates every extensible part of their configuration. --- flake8 => .flake8 | 0 Makefile | 2 +- README.mkd | 12 +- pylintrc | 3 + setup.py | 1 + tests/invalid-thumbor.conf | 48 ++++ tests/snapshots/__init__.py | 0 tests/snapshots/snap_test_doctor.py | 171 +++++++++++++ tests/test_doctor.py | 26 ++ thumbor/doctor.py | 378 ++++++++++++++++++++++------ 10 files changed, 563 insertions(+), 78 deletions(-) rename flake8 => .flake8 (100%) create mode 100644 tests/invalid-thumbor.conf create mode 100644 tests/snapshots/__init__.py create mode 100644 tests/snapshots/snap_test_doctor.py create mode 100644 tests/test_doctor.py diff --git a/flake8 b/.flake8 similarity index 100% rename from flake8 rename to .flake8 diff --git a/Makefile b/Makefile index a458c2cb1..23ea5da06 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ format: @black . flake: - @flake8 --config flake8 + @flake8 --config .flake8 pylint: @pylint thumbor tests diff --git a/README.mkd b/README.mkd index 2678fb326..9ca367588 100644 --- a/README.mkd +++ b/README.mkd @@ -93,7 +93,17 @@ If you experience any troubles, try running: thumbor-doctor ``` -If you still need help, please [raise an issue](https://github.com/thumbor/thumbor/issues). +If you have a `thumbor.conf` file, you can use that to help thumbor-doctor: + +```bash +thumbor-doctor -c thumbor.conf +``` + +If you still need help, please [raise an issue](https://github.com/thumbor/thumbor/issues). Remember to send your `thumbor-doctor` output in the issue: + +```bash +thumbor-doctor --nocolor -c thumbor.conf +``` ## 🎯 Features diff --git a/pylintrc b/pylintrc index b245bc90f..721302168 100644 --- a/pylintrc +++ b/pylintrc @@ -1,3 +1,6 @@ +[MASTER] +ignore=tests/snapshots + [MESSAGES CONTROL] # Disable the message, report, category or checker with the given id(s). You diff --git a/setup.py b/setup.py index 1ca0b2efb..a529ec3f5 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ "redis==3.*,>=3.4.0", "remotecv>=2.3.0", "sentry-sdk==0.*,>=0.14.1", + "snapshottest>=0.6.0,<1.0.0", "yanc==0.*,>=0.3.3", ] diff --git a/tests/invalid-thumbor.conf b/tests/invalid-thumbor.conf new file mode 100644 index 000000000..b99b62551 --- /dev/null +++ b/tests/invalid-thumbor.conf @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# thumbor imaging service +# https://github.com/thumbor/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com thumbor@googlegroups.com + +LOADER = "thumbor.loaders.http_loaderer" + +UPLOAD_ENABLED = True +UPLOAD_PHOTO_STORAGE = "thumbor.storages.file_storager" + +STORAGE = "thumbor.storages.file_storagee" + +RESULT_STORAGE = "thumbor.result_storages.file_storagee" + +STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True + +ENGINE = "thumbor.engines.pillage" + +## detectors to use to find Focal Points in the image +## more about detectors can be found in thumbor's docs +## at https://github.com/thumbor/thumbor/wiki +DETECTORS = [ + "thumbor.detectors.face_detectorer", + "thumbor.detectors.other_invalid", +] + +USE_CUSTOM_ERROR_HANDLING = True +ERROR_HANDLER_MODULE = "thumbor.error_handlers.sentryer" + +FILTERS = [ + "invalid-filter", +] + +OPTIMIZERS = [ + "thumbor.optimizers.jpegtraner", + "thumbor.optimizers.gifver", +] + +from thumbor.handler_lists import BUILTIN_HANDLERS + +HANDLER_LISTS = BUILTIN_HANDLERS + [ + 'my.invalid.handler', +] + +ALLOW_UNSAFE_URL = True diff --git a/tests/snapshots/__init__.py b/tests/snapshots/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/snapshots/snap_test_doctor.py b/tests/snapshots/snap_test_doctor.py new file mode 100644 index 000000000..168101f71 --- /dev/null +++ b/tests/snapshots/snap_test_doctor.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# snapshottest: v1 - https://goo.gl/zC4yUc +from __future__ import unicode_literals + +from snapshottest import Snapshot + + +snapshots = Snapshot() + +snapshots['test_get_doctor_output 1'] = '''Using configuration file found at ./tests/invalid-thumbor.conf + +Thumbor doctor will analyze your install and verify if everything is working as expected. + +Verifying libraries support... + +βœ… pycurl is installed correctly. +βœ… cairosvg is installed correctly. +βœ… cv2 is installed correctly. + +Verifying thumbor compiled extensions... + +βœ… _alpha +βœ… _bounding_box +βœ… _brightness +βœ… _colorize +βœ… _composite +βœ… _contrast +βœ… _convolution +βœ… _curve +βœ… _equalize +βœ… _fill +βœ… _nine_patch +βœ… _noise +βœ… _rgb +βœ… _round_corner +βœ… _saturation +βœ… _sharpen + +Verifying extensibility modules found in your thumbor.conf... + +❎ thumbor.storages.file_storagee - Storage for source images could not be imported. +❎ thumbor.loaders.http_loaderer - Loader for source images could not be imported. +❎ thumbor.result_storages.file_storagee - ResultStorage could not be imported. +❎ thumbor.engines.pillage - Engine for transforming images could not be imported. +❎ thumbor.storages.file_storager - Uploading to thumbor is enabled and the Upload Storage could not be imported. +❎ thumbor.detectors.face_detectorer - Detector could not be imported. +❎ thumbor.detectors.other_invalid - Detector could not be imported. +❎ invalid-filter - Filter could not be imported. +❎ thumbor.optimizers.jpegtraner - Optimizer could not be imported. +❎ thumbor.optimizers.gifver - Optimizer could not be imported. +❎ thumbor.error_handlers.sentryer - Custom error handling is enabled and the error handler module could not be imported. +βœ… thumbor.handler_lists.healthcheck +βœ… thumbor.handler_lists.upload +βœ… thumbor.handler_lists.blacklist +❎ my.invalid.handler - Custom http handler could not be imported. + +Verifying security... + +❎ Using default security key. +❎ Allowing unsafe URLs. + +😞 Oh no! We found some things that could improve... 😞 + +⚠️Warnings⚠️ +* Security + Error Message: + Using default security key configuration in thumbor.conf. + + Error Description: + You should specify a unique security key for thumbor or use a command line param to specify a security key. +\tFor more information visit https://thumbor.readthedocs.io/en/latest/running.html + +β›”Errorsβ›” +* thumbor.storages.file_storagee + Error Message: + No module named 'thumbor.storages.file_storagee' + + Error Description: + Storage for source images could not be imported. + +* thumbor.loaders.http_loaderer + Error Message: + No module named 'thumbor.loaders.http_loaderer' + + Error Description: + Loader for source images could not be imported. + +* thumbor.result_storages.file_storagee + Error Message: + No module named 'thumbor.result_storages.file_storagee' + + Error Description: + ResultStorage could not be imported. + +* thumbor.engines.pillage + Error Message: + No module named 'thumbor.engines.pillage' + + Error Description: + Engine for transforming images could not be imported. + +* thumbor.storages.file_storager + Error Message: + No module named 'thumbor.storages.file_storager' + + Error Description: + Uploading to thumbor is enabled and the Upload Storage could not be imported. + +* thumbor.detectors.face_detectorer + Error Message: + No module named 'thumbor.detectors.face_detectorer' + + Error Description: + Detector could not be imported. + +* thumbor.detectors.other_invalid + Error Message: + No module named 'thumbor.detectors.other_invalid' + + Error Description: + Detector could not be imported. + +* invalid-filter + Error Message: + No module named 'invalid-filter' + + Error Description: + Filter could not be imported. + +* thumbor.optimizers.jpegtraner + Error Message: + No module named 'thumbor.optimizers.jpegtraner' + + Error Description: + Optimizer could not be imported. + +* thumbor.optimizers.gifver + Error Message: + No module named 'thumbor.optimizers.gifver' + + Error Description: + Optimizer could not be imported. + +* thumbor.error_handlers.sentryer + Error Message: + No module named 'thumbor.error_handlers.sentryer' + + Error Description: + Custom error handling is enabled and the error handler module could not be imported. + +* my.invalid.handler + Error Message: + No module named 'my' + + Error Description: + Custom http handler could not be imported. + +* Security + Error Message: + Unsafe URLs are enabled. + + Error Description: + It is STRONGLY recommended that you turn off ALLOW_UNSAFE_URLS flag in production environments as this can lead to DDoS attacks against thumbor. +\tFor more information visit https://thumbor.readthedocs.io/en/latest/security.html + +❓Need Help❓ + +If you don't know how to fix the above problems, please open an issue with thumbor. +Don't forget to copy this log and add it to the description. +Open an issue at https://github.com/thumbor/thumbor/issues/new +''' diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 000000000..568048418 --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,26 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# thumbor imaging service +# https://github.com/thumbor/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com thumbor@googlegroups.com + + +from thumbor.doctor import run_doctor + + +def test_get_doctor_output(snapshot, capsys): + run_doctor( + { + "nocolor": True, + "config": "./tests/invalid-thumbor.conf", + }, + print_version=False, + exit_with_error=False, + check_pyexiv=False, + ) + result = capsys.readouterr() + snapshot.assert_match(result.out) diff --git a/thumbor/doctor.py b/thumbor/doctor.py index c915ec292..58c345d87 100644 --- a/thumbor/doctor.py +++ b/thumbor/doctor.py @@ -8,21 +8,25 @@ # http://www.opensource.org/licenses/mit-license # Copyright (c) 2011 globo.com thumbor@googlegroups.com -# pylint: disable=no-member +# pylint: disable=no-member,line-too-long import argparse import sys from importlib import import_module +from os.path import abspath from shutil import which import colorful as cf from thumbor import __release_date__, __version__ +from thumbor.config import Config from thumbor.ext import BUILTIN_EXTENSIONS -from thumbor.filters import BUILTIN_FILTERS CHECK = "βœ…" -CROSS = "❎ " +CROSS = "❎" +WARNING = "⚠️" +ERROR = "β›”" +QUESTION = "❓" def get_options(): @@ -35,10 +39,21 @@ def get_options(): help="Disables coloring of thumbor doctor", ) + parser.add_argument( + "-c", + "--config", + default=None, + help=( + "thumbor configuration file. If specified " + "thumbor-doctor can be fine tuned." + ), + ) + options = parser.parse_args() return { "nocolor": options.nocolor, + "config": options.config, } @@ -55,18 +70,85 @@ def newline(): print() -def check_filters(): +def check_extensibility_modules(cfg): + if cfg is None: + return None + newline() - subheader("Verifying thumbor filters...") errors = [] - for filter_name in BUILTIN_FILTERS: - try: - import_module(filter_name) - print(cf.bold_green(f"{CHECK} {filter_name}")) - except ImportError as error: - print(cf.bold_red(f"{CROSS} {filter_name}")) - errors.append(error) + to_check = [ + ( + lambda cfg: True, + cfg.STORAGE, + "Storage for source images could not be imported.", + ), + ( + lambda cfg: True, + cfg.LOADER, + "Loader for source images could not be imported.", + ), + (lambda cfg: True, cfg.RESULT_STORAGE, "ResultStorage could not be imported."), + ( + lambda cfg: True, + cfg.ENGINE, + "Engine for transforming images could not be imported.", + ), + ( + lambda cfg: cfg.UPLOAD_ENABLED, + cfg.UPLOAD_PHOTO_STORAGE, + "Uploading to thumbor is enabled and the Upload Storage could not be imported.", + ), + ( + lambda cfg: True, + cfg.DETECTORS, + "Detector could not be imported.", + ), + ( + lambda cfg: True, + cfg.FILTERS, + "Filter could not be imported.", + ), + ( + lambda cfg: True, + cfg.OPTIMIZERS, + "Optimizer could not be imported.", + ), + ( + lambda cfg: cfg.USE_CUSTOM_ERROR_HANDLING, + cfg.ERROR_HANDLER_MODULE, + "Custom error handling is enabled and the " + "error handler module could not be imported.", + ), + ( + lambda cfg: True, + cfg.HANDLER_LISTS, + "Custom http handler could not be imported.", + ), + ] + + if any(c[0](cfg) for c in to_check): + subheader("Verifying extensibility modules found in your thumbor.conf...") + + for should_check, modules, error_message in to_check: + if not should_check(cfg): + continue + if not isinstance(modules, (list, tuple)): + modules = [modules] + + for module in modules: + try: + import_module(module) + print(cf.bold_green(f"{CHECK} {module}")) + except ImportError as error: + print(cf.bold_red(f"{CROSS} {module} - {error_message}")) + errors.append( + format_error( + module, + str(error), + error_message, + ) + ) return errors @@ -83,37 +165,61 @@ def check_compiled_extensions(): print(cf.bold_green(f"{CHECK} {ext_name}")) except ImportError as error: print(cf.bold_red(f"{CROSS} {ext_name}")) - errors.append(error) + errors.append( + format_error( + f"Extension {extension}", + str(error), + ( + "Extension could not be compiled. " + "This will lead to filter being disabled." + ), + ) + ) return errors -def check_modules(): +def format_error(dependency, err, msg): + formatted_msg = "\n\t".join(msg.split("\n")) + result = f""" +* {dependency} + Error Message: + {err} + + Error Description: + { formatted_msg } + """ + return result.strip() + + +def check_modules(cfg, check_pyexiv=True): newline() - subheader("Verifying libraries support...") errors = [] - modules = ( + modules = [ ( "pycurl", - "Thumbor works much better with PyCurl. For more information visit http://pycurl.io/.", - ), - ( - "cv2", - "Thumbor requires OpenCV for smart cropping. " - "For more information check https://opencv.org/.", - ), - ( - "pyexiv2", - "Thumbor uses exiv2 for reading image metadata. " - "For more information check https://python3-exiv2.readthedocs.io/en/latest/.", + "Thumbor works much better with PyCurl. " + "For more information visit http://pycurl.io/.", ), ( "cairosvg", "Thumbor uses CairoSVG for reading SVG files. " "For more information check https://cairosvg.org/.", ), - ) + ] + + if cfg is None or len(cfg.DETECTORS) != 0: + modules.append( + ( + "cv2", + "Thumbor requires OpenCV for smart cropping. " + "For more information check https://opencv.org/.", + ) + ) + + if modules: + subheader("Verifying libraries support...") for module, error_message in modules: try: @@ -123,34 +229,74 @@ def check_modules(): print(cf.bold_red(f"{CROSS} {module} is not installed.")) print(error_message) newline() - errors.append(f"{str(error)} - {error_message}") + errors.append(format_error(module, str(error), error_message)) + + warn_modules = [] + + if check_pyexiv: + warn_modules.append( + ( + "pyexiv2", + ( + "If you do not need EXIF metadata, you can safely ignore this.\n" + "For more information visit " + "https://python3-exiv2.readthedocs.io/en/latest/." + ), + ), + ) - return errors + warnings = [] + for module, error_message in warn_modules: + try: + import_module(module) # NOQA + print(cf.bold_green(f"{CHECK} {module} is installed correctly.")) + except ImportError as error: + print(cf.bold_yellow(f"{CROSS} {module} is not installed.")) + print(error_message) + newline() + warnings.append(format_error(module, str(error), error_message)) + return warnings, errors -def check_extensions(): + +def check_extensions(cfg): newline() - subheader("Verifying extension programs...") errors = [] + programs = [] + + if cfg is None or "thumbor.optimizers.jpegtran" in cfg.OPTIMIZERS: + programs.append( + ( + "jpegtran", + ( + "Thumbor uses jpegtran for optimizing JPEG images. " + "For more information visit " + "https://linux.die.net/man/1/jpegtran." + ), + ) + ) - programs = ( - ( - "jpegtran", - "Thumbor uses jpegtran for optimizing JPEG images. " - "For more information visit https://linux.die.net/man/1/jpegtran.", - ), - ( - "ffmpeg", - "Thumbor uses ffmpeg for rendering animated images as GIFV. " - "For more information visit https://www.ffmpeg.org/.", - ), - ( - "gifsicle", - "Thumbor uses gifsicle for better processing of GIF images. " - "For more information visit https://www.lcdf.org/gifsicle/.", - ), - ) + if cfg is None or "thumbor.optimizers.gifv" in cfg.OPTIMIZERS: + programs.append( + ( + "ffmpeg", + "Thumbor uses ffmpeg for rendering animated images as GIFV. " + "For more information visit https://www.ffmpeg.org/.", + ) + ) + + if cfg is None or cfg.USE_GIFSICLE_ENGINE: + programs.append( + ( + "gifsicle", + "Thumbor uses gifsicle for better processing of GIF images. " + "For more information visit https://www.lcdf.org/gifsicle/.", + ) + ) + + if programs: + subheader("Verifying extension programs...") for program, error_message in programs: path = which(program) @@ -158,58 +304,138 @@ def check_extensions(): print(cf.bold_red(f"{CROSS} {program} is not installed.")) print(error_message) newline() - errors.append(error_message) + errors.append( + format_error(program, f"Could not find {program}.", error_message) + ) else: print(cf.bold_green(f"{CHECK} {program} is installed correctly.")) return errors -def main(): - """Converts a given url with the specified arguments.""" +def check_security(cfg): + subheader("Verifying security...") + errors = [] + warnings = [] + + if cfg.SECURITY_KEY == "MY_SECURE_KEY": + print(cf.bold_red(f"{CROSS} Using default security key.")) + + warnings.append( + format_error( + "Security", + "Using default security key configuration in thumbor.conf.", + "You should specify a unique security key for thumbor or " + "use a command line param to specify a security key.\n" + "For more information visit " + "https://thumbor.readthedocs.io/en/latest/running.html", + ) + ) + + if cfg.ALLOW_UNSAFE_URL: + print(cf.bold_red(f"{CROSS} Allowing unsafe URLs.")) + + errors.append( + format_error( + "Security", + "Unsafe URLs are enabled.", + "It is STRONGLY recommended that you turn off " + "ALLOW_UNSAFE_URLS flag in production environments " + "as this can lead to DDoS attacks against thumbor.\n" + "For more information visit " + "https://thumbor.readthedocs.io/en/latest/security.html", + ) + ) + + return errors, warnings - options = get_options() +def load_config(config_path): + cfg = None + if config_path is not None: + path = abspath(config_path) + cfg = Config.load(path, conf_name="thumbor.conf", lookup_paths=[]) + print(f"Using configuration file found at {config_path}") + return cfg + + +def configure_colors(nocolor): cf.use_style("solarized") - if options["nocolor"]: + if nocolor: cf.disable() - newline() - header(f"Thumbor v{__version__} (of {__release_date__})") +def print_header(print_version=True): + if print_version: + newline() + header(f"Thumbor v{__version__} (of {__release_date__})") newline() print( - "Thumbor doctor will analyze your install and verify if everything is working as expected." + "Thumbor doctor will analyze your install and verify " + "if everything is working as expected." ) - errors = check_modules() + +def check_everything(cfg, check_pyexiv): + warnings, errors = check_modules(cfg, check_pyexiv) errors += check_compiled_extensions() - errors += check_filters() - errors += check_extensions() + errors += check_extensibility_modules(cfg) + errors += check_extensions(cfg) + + sec_err, sec_warn = check_security(cfg) + errors += sec_err + warnings += sec_warn newline() + return warnings, errors - if errors: - print(cf.bold_red("😞 Oh no! We found some things that could improve... 😞")) - newline() - print("\n".join([f"* {str(err)}" for err in errors])) + +def print_results(warnings, errors): + if not warnings and not errors: + print(cf.bold_green("πŸŽ‰ Congratulations! No errors found! πŸŽ‰")) + sys.exit(1) + return + + print(cf.bold_red("😞 Oh no! We found some things that could improve... 😞")) + newline() + + if warnings: + print(cf.bold_yellow(f"{WARNING}Warnings{WARNING}")) + print("\n\n".join([f"{str(err)}" for err in warnings])) newline() + + if errors: + print(cf.bold_red(f"{ERROR}Errors{ERROR}")) + print("\n\n".join([f"{str(err)}" for err in errors])) newline() - print( - cf.cyan( - "If you don't know how to fix them, please open an issue with thumbor." - ) - ) - print( - cf.cyan( - "Don't forget to copy this log and add it to the description of your issue." - ) + + subheader(f"{QUESTION}Need Help{QUESTION}") + print( + cf.cyan( + "If you don't know how to fix the above problems, please open an issue with thumbor." ) - print("Open an issue at https://github.com/thumbor/thumbor/issues/new") + ) + print(cf.cyan("Don't forget to copy this log and add it to the description.")) + print("Open an issue at https://github.com/thumbor/thumbor/issues/new") + + +def run_doctor(options, print_version=True, exit_with_error=True, check_pyexiv=True): + cfg = load_config(options["config"]) + configure_colors(options["nocolor"]) + + print_header(print_version) + warnings, errors = check_everything(cfg, check_pyexiv) + print_results(warnings, errors) + + if exit_with_error and errors: sys.exit(1) - return - print(cf.bold_green("πŸŽ‰ Congratulations! No errors found! πŸŽ‰")) + +def main(): + """Verifies the current environment for problems with thumbor""" + + options = get_options() + run_doctor(options, print_version=True, exit_with_error=True) if __name__ == "__main__":