Merge pull request #2 from thumbor/to-py3

migrate thumborizeme to python3

Makefile

@@ -1,2 +1,5 @@
 run:
-	@cd thumborizeme && python app.py 9000
+	@python ./thumborizeme/server.py
+
+setup:
+	@pip install -Ue .

README.md

@@ -1,2 +1,45 @@
-thumborizeme
-============
+# Thumborizeme
+
+Thumborize is an application that shows the benefits of using the
+[Thumbor](https://github.com/thumbor/thumbor).
+
+## Configuration
+
+To use redis some environments variables must be configured
+
+
+##### Single Node
+```python
+REDIS_HOST = "localhost"
+REDIS_DB = 0
+REDIS_PORT = 6379
+REDIS_PASSWORD = None
+REDIS_MODE = "single_node"
+```
+
+##### Sentinel
+```python
+REDIS_SENTINEL_MASTER_INSTANCE = "redismaster"
+REDIS_SENTINEL_MASTER_DB = 0
+REDIS_SENTINEL_MASTER_PASSWORD = "dummy"
+REDIS_SENTINEL_INSTANCES = "localhost:26379,localhost:26380"
+REDIS_SENTINEL_PASSWORD = "dummy"
+REDIS_SENTINEL_SOCKET_TIMEOUT = 1.0
+REDIS_MODE = "sentinel"
+```
+
+## Dependencies
+
+To install the dependencies, run the commands bellow:
+
+```sh
+make setup
+```
+
+## Run locally
+
+To start thumborizeme, run the commands bellow:
+
+```sh
+make run
+```

setup.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from setuptools import setup, find_packages
+
+__version__ = "0.0.1+globo.2"
+
+RUNTIME_REQUIREMENTS = [
+    "black==22.*,>=22.1.0",
+    "cssselect==1.1.0",
+    "gunicorn==20.1.0",
+    "lxml==4.9.1",
+    "numpy==1.23.0",
+    "pycurl==7.45.1",
+    "pydantic==1.10.2",
+    "redis==4.3.4",
+    "requests==2.28.1",
+    "thumbor==7.1.0",
+    "tornado==6.2",
+    "pydantic",
+    "pre-commit==2.*,>=2.17.0",
+]
+
+setup(
+    name="thumborizeme",
+    version=__version__,
+    description="thumborize app",
+    author="Bernardo Heynemann",
+    author_email="[email protected]",
+    url="https://github.com/heynemann/thumborizeme",
+    license="MIT",
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Developers",
+        "Natural Language :: English",
+        "Operating System :: MacOS",
+        "Operating System :: POSIX :: Linux",
+        "Programming Language :: Python :: 3",
+        "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
+        "Topic :: Multimedia :: Graphics :: Presentation",
+    ],
+    packages=find_packages(),
+    package_data={
+        "thumborizeme": [
+            "static/*",
+            "static/css/*",
+            "static/images/*",
+            "static/js/*",
+            "static/fonts/*",
+        ],
+    },
+    include_package_data=True,
+    install_requires=RUNTIME_REQUIREMENTS,
+    entry_points={
+        "console_scripts": [
+            "thumborizeme=thumborizeme.server:main",
+        ],
+    },
+)

thumborizeme/app.py

@@ -1,173 +1,31 @@
-import sys
-import os.path
-from json import dumps, loads
-from datetime import datetime
-
-import lxml.html
-
-import tornado.ioloop
 import tornado.web
-import tornado.gen
-
-from tornado.httpclient import AsyncHTTPClient, HTTPRequest
-from tornado.concurrent import return_future
-
-from toredis import Client
-
-import settings
-
-root_path = os.path.abspath(os.path.join(os.path.dirname(__file__)))
-AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
-io_loop = tornado.ioloop.IOLoop.instance()
-
-
-class HealthCheckHandler(tornado.web.RequestHandler):
-    @tornado.gen.coroutine
-    def get(self):
-        try:
-            yield tornado.gen.Task(self.application.redis.ping)
-            self.write("WORKING !")
-        except Exception:
-            self.set_status(500)
-            self.write("DOWN !")
-        self.finish()
-
-
-class MainHandler(tornado.web.RequestHandler):
-    @tornado.gen.coroutine
-    def get(self):
-        url = self.get_argument('url', None)
-
-        if url is None:
-            title = "Check how you would benefit from using thumbor"
-        else:
-            title = "Test results for %s" % url
-
-        total_images = yield tornado.gen.Task(self.application.redis.get, 'total_images')
-        total_images = int(total_images or 0)
-
-        year = datetime.now().year
-
-        self.render('index.html', title=title, total_images=total_images, year=year, host=settings.HOST, thumbor_host=settings.THUMBOR_HOST)
-
-
-class GetReportHandler(tornado.web.RequestHandler):
-    @tornado.gen.coroutine
-    def get(self):
-        site_url = self.get_argument('url')
-
-        cached_data = yield tornado.gen.Task(self.application.redis.get, site_url.rstrip('/'))
-        if cached_data is not None:
-            print "GETTING FROM CACHE"
-            self.write(loads(cached_data))
-            self.finish()
-            return
-
-        response = yield self.get_content(site_url)
-        html = lxml.html.fromstring(response.body)
-        imgs = html.cssselect('img[src]')
-
-        images = {}
-        for img in imgs:
-            url = img.get('src').lstrip('//')
-
-            if not url.startswith('http'):
-                url = "%s/%s" % (site_url.rstrip('/'), url)
-                if 'data:image' in url:
-                    continue
-
-            print "Loading %s..." % url
-
-            try:
-                if url in images:
-                    continue
-
-                loaded = yield self.get_content(url)
-
-                if loaded.code != 200:
-                    continue
-                original_size = len(loaded.body)
-
-                webp = "%s/unsafe/filters:strip_icc():format(webp):quality(80)/%s" % (settings.THUMBOR_HOST, url)
-                webp_loaded = yield self.get_content(webp)
-
-                if webp_loaded.code != 200:
-                    continue
-                webp_size = len(webp_loaded.body)
-
-                images[url] = {
-                    'original': original_size / 1024.0,
-                    # 'thumborized': thumborized_size / 1024.0,
-                    'webp': webp_size / 1024.0
-                }
-
-            except Exception, err:
-                print str(err)
-                continue
-
-        yield tornado.gen.Task(self.application.redis.incrby, 'total_images', len(images.keys()))
-
-        json_data = self.to_json({
-            'url': site_url,
-            'images-count': len(images.keys()),
-            'images-size': round(sum([image['original'] for image in images.values()]), 2),
-            'images-webp-size': round(sum([image['webp'] for image in images.values()]), 2)
-        })
-
-        yield tornado.gen.Task(self.application.redis.setex, site_url.rstrip('/'), 6 * 60 * 60, json_data)
-
-        self.write(json_data)
-
-        self.finish()
-
-    def to_json(self, value):
-        return dumps(value)
-
-    @return_future
-    def get_content(self, url, callback):
-        proxy_host = settings.PROXY_HOST_HTTPS if url.startswith('https') else settings.PROXY_HOST
-        req = HTTPRequest(
-            url=url,
-            connect_timeout=3,
-            request_timeout=10,
-            proxy_host=proxy_host,
-            proxy_port=int(settings.PROXY_PORT)
-        )
-
-        http_client = AsyncHTTPClient()
-        http_client.fetch(req, callback)
-
-    def is_expired(self, dt):
-        return (datetime.now() - dt).total_seconds() > (6 * 60 * 60)
-
-
-def has_connected(application, io_loop):
-    def handle(*args, **kw):
-        pass
-
-    return handle
-
-
-application = tornado.web.Application([
-    (r"/", MainHandler),
-    (r"/report", GetReportHandler),
-    (r"/healthcheck", HealthCheckHandler),
-], static_path=root_path, template_path=root_path)
-
-redis_host = settings.REDIS_HOST
-redis_port = int(settings.REDIS_PORT)
-
-application.redis = Client(io_loop=io_loop)
-application.redis.authenticated = True
-application.redis.connect(redis_host, redis_port, callback=has_connected(application, io_loop))
-application.redis.auth(settings.REDIS_PASSWORD)
-
-
-if __name__ == "__main__":
-    if len(sys.argv) > 2:
-        server_port = int(sys.argv[2])
-    else:
-        server_port = int(sys.argv[1])
-
-    application.listen(server_port, address='0.0.0.0')
-    io_loop.start()
+import os
+
+from thumborizeme.settings import Settings
+from thumborizeme.handlers.report import ReportHandler
+from thumborizeme.handlers.healthcheck import HealthCheckHandler
+from thumborizeme.handlers.home import HomeHandler
+from thumborizeme.redis_client import RedisClient
+
+
+class ThumborizemeApp(tornado.web.Application):
+    def __init__(self):
+        self.config = Settings()
+        self.redis_client = RedisClient(self.config).initialize()
+
+        root = os.path.dirname(__file__)
+        handlers = [
+            (
+                r"/",
+                HomeHandler,
+            ),
+            (r"/report", ReportHandler),
+            (r"/healthcheck", HealthCheckHandler),
+            (
+                "/static/(.*)",
+                tornado.web.StaticFileHandler,
+                {"path": root + "/static"},
+            ),
+        ]
+
+        super(ThumborizemeApp, self).__init__(handlers, static_path=root + "/static")

thumborizeme/handlers/healthcheck.py

@@ -0,0 +1,17 @@
+from tornado.web import RequestHandler
+
+
+class HealthCheckHandler(RequestHandler):
+    def __init__(self, application, request, **kwargs):
+        self.redis_client = application.redis_client
+
+        super(HealthCheckHandler, self).__init__(application, request, **kwargs)
+
+    async def get(self):
+        try:
+            self.redis_client.ping()
+            self.write("WORKING !")
+        except Exception:
+            self.set_status(500)
+            self.write("DOWN !")
+        self.finish()

thumborizeme/handlers/home.py

@@ -0,0 +1,33 @@
+from datetime import datetime
+
+from tornado.web import RequestHandler
+
+
+class HomeHandler(RequestHandler):
+    def __init__(self, application, request, **kwargs):
+        self.redis_client = application.redis_client
+        self.config = application.config
+
+        super(HomeHandler, self).__init__(application, request, **kwargs)
+
+    async def get(self):
+        url = self.get_argument("url", None)
+
+        if url is None:
+            title = "Check how you would benefit from using thumbor"
+        else:
+            title = f"Test results for {url}"
+
+        total_images = self.redis_client.get("total_images")
+        total_images = int(total_images or 0)
+
+        year = datetime.now().year
+
+        self.render(
+            "../static/index.html",
+            title=title,
+            total_images=total_images,
+            year=year,
+            host=self.config.get("HOST"),
+            thumbor_host=self.config.get("THUMBOR_HOST"),
+        )