Merge pull request #9 from tilezen/zerebubuth/size-heatmap

Add heatmap server

README.md

@@ -5,6 +5,7 @@ Current scoville commands:
 * `info`: Prints size and item count information about an MVT tile.
 * `proxy`: Serves a treemap visualisation of tiles on a local HTTP server.
 * `percentiles`: Calculate the percentile tile sizes for a set of MVT tiles.
+* `heatmap`: Serves a heatmap visualisation of tile sizes on a local HTTP server.
 
 ### Info command ###
 
@@ -96,6 +97,35 @@ Will output something like:
 
 Note that the `~total` entry is **not** the total of the column above it; it's the percentile of total tile size. In other words, if we had three tiles with three layers, and each tile had a single, different layer taking up 1000 bytes and two layers taking up 10 bytes, then each tile is 1020 bytes and that would be the p50 `~total`. However, the p50 on each individual layer would only be 10 bytes.
 
+
+### Heatmap command ###
+
+Runs a local tile server showing heatmap tiles. The darker the colour, the larger the tile is. Each 256x256 tile is made up of an 8x8 heat grid, so the smaller squares you'll see represent tiles at a zoom 3 levels down from the level you're viewing at. This stops at z16, so you'll see the squares getting coarser as you approach that zoom.
+
+The mapping between tile size and colour is:
+
+ * <50kB: `#ffffff`
+ * <100kB: `#fff7ec`
+ * <150kB: `#fee8c8`
+ * <200kB: `#fdd49e`
+ * <250kB: `#fdbb84`
+ * <300kB: `#fc8d59`
+ * <500kB: `#ef6548`
+ * <750kB: `#d7301f`
+ * <1000kB: `#990000`
+ * >=1000kB: `#000000`
+
+To run it:
+
+```
+scoville heatmap "https://tile.nextzen.org/tilezen/vector/v1/512/all/{z}/{x}/{y}.mvt?api_key=YOUR_API_KEY"
+```
+
+This will run a server on [localhost:8000](http://localhost:8000) by default (use `--port` option to change the port). Navigating to that page should show you something like:
+
+![Screenshot of the heatmap server](doc/heatmap_screenshot.png)
+
+
 ## Install on Ubuntu:
 
 ```

scoville/command.py

@@ -135,8 +135,8 @@ def proxy(url, port):
     URL should contain {z}, {x} and {y} replacements.
     """
 
-    from scoville.proxy import serve_http
-    serve_http(url, port)
+    from scoville.proxy import serve_http, Treemap
+    serve_http(url, port, Treemap())
 
 
 def read_urls(file_name, url_pattern):
@@ -234,6 +234,46 @@ def percentiles(tiles_file, url, percentiles, cache, nprocs, output_format):
         raise ValueError('Unknown output format %r' % (output_format,))
 
 
+@cli.command()
+@click.argument('url', required=1)
+@click.option('--port', default=8000, help='Port to serve tiles on.')
+def heatmap(url, port):
+    """
+    Serves a heatmap of tile sizes on localhost:PORT.
+
+    URL should contain {z}, {x} and {y} replacements.
+    """
+
+    from scoville.proxy import serve_http, Heatmap
+
+    def colour_map(size):
+        kb = size / 1024
+
+        if kb < 6:
+            return '#ffffff'
+        elif kb < 12:
+            return '#fff7ec'
+        elif kb < 25:
+            return '#fee8c8'
+        elif kb < 50:
+            return '#fdd49e'
+        elif kb < 75:
+            return '#fdbb84'
+        elif kb < 125:
+            return '#fc8d59'
+        elif kb < 250:
+            return '#ef6548'
+        elif kb < 500:
+            return '#d7301f'
+        elif kb < 750:
+            return '#990000'
+        else:
+            return '#000000'
+
+    heatmap = Heatmap(3, 16, colour_map)
+    serve_http(url, port, heatmap)
+
+
 def scoville_main():
     cli()
 

scoville/proxy.py

@@ -10,60 +10,30 @@
 TILE_PATTERN = re.compile('^/tiles/([0-9]+)/([0-9]+)/([0-9]+)\.png$')
 
 
-class Handler(http.server.BaseHTTPRequestHandler):
-    def do_GET(self):
-        if self.path in ("/", "/index.html", "/style.css", "/map.js"):
-            template_name = self.path[1:]
-            if not template_name:
-                template_name = "index.html"
-
-            self.send_template(template_name)
-            return
-
-        m = TILE_PATTERN.match(self.path)
-        if m:
-            z, x, y = map(int, m.groups())
-
-            if 0 <= z < 16 and \
-               0 <= x < (1 << z) and \
-               0 <= y < (1 << z):
-                self.send_tile_breakdown(z, x, y)
-                return
-
-        self.error_not_found()
+class Treemap(object):
+    """
+    Draws a Treemap of layer sizes within the tile.
+    """
 
-    def error_not_found(self):
-        self.send_response(requests.codes.not_found)
-
-    def send_tile_breakdown(self, z, x, y):
-        url = self.server.url_pattern \
-                         .replace("{z}", str(z)) \
-                         .replace("{x}", str(x)) \
-                         .replace("{y}", str(y))
+    def tiles_for(self, z, x, y):
+        return {0: (z, x, y)}
 
-        res = requests.get(url)
-        if res.status_code != requests.codes.ok:
-            self.send_response(res.status_code)
-            return
+    def render(self, tiles):
+        from PIL import Image, ImageDraw, ImageFont
 
-        tile = Tile(res.content)
+        tile = tiles[0]
         sizes = []
         for layer in tile:
             sizes.append((layer.size, layer.name))
         sizes.sort(reverse=True)
 
-        self.send_png(sizes)
-
-    def send_png(self, sizes):
-        from PIL import Image, ImageDraw, ImageFont
-
         width = height = 256
+        im = Image.new("RGB", (width, height), "black")
 
         values = squarify.normalize_sizes([r[0] for r in sizes], width, height)
         rects = squarify.squarify(values, 0, 0, width, height)
         names = [r[1] for r in sizes]
 
-        im = Image.new("RGB", (width, height), "black")
         draw = ImageDraw.Draw(im)
         font = ImageFont.load_default()
 
@@ -87,8 +57,113 @@ def send_png(self, sizes):
                 draw.text(top_left, name, fill='black', font=font)
 
         del draw
+        return im
+
+
+class Heatmap(object):
+    """
+    Renders each tile as a heatmap.
+    """
+
+    def __init__(self, sub_zooms, max_zoom, colour_map):
+        self.sub_zooms = sub_zooms
+        self.max_zoom = max_zoom
+        self.colour_map = colour_map
+
+    def tiles_for(self, z, x, y):
+        sub_z = min(z + self.sub_zooms, self.max_zoom)
+        dz = sub_z - z
+        width = 1 << dz
+        tiles = {}
+        for dx in xrange(0, width):
+            for dy in xrange(0, width):
+                tiles[(dx, dy)] = (sub_z, (x << dz) + dx, (y << dz) + dy)
+        return tiles
+
+    def render(self, tiles):
+        from PIL import Image, ImageDraw
+
+        max_coord = max(tiles.keys())
+        assert max_coord[0] == max_coord[1]
+        ntiles = max_coord[0] + 1
+        assert len(tiles) == ntiles ** 2
+
+        width = height = 256
+        im = Image.new("RGB", (width, height), "black")
+
+        draw = ImageDraw.Draw(im)
+
+        scale = width / ntiles
+        assert width == scale * ntiles
+
+        for x in xrange(0, ntiles):
+            for y in xrange(0, ntiles):
+                size = len(tiles[(x, y)].data)
+                colour = self.colour_map(size)
+
+                draw.rectangle(
+                    [x * scale, y * scale, (x+1) * scale, (y+1) * scale],
+                    fill=colour)
+
+        del draw
+        return im
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path in ("/", "/index.html", "/style.css", "/map.js"):
+            template_name = self.path[1:]
+            if not template_name:
+                template_name = "index.html"
+
+            self.send_template(template_name)
+            return
+
+        m = TILE_PATTERN.match(self.path)
+        if m:
+            z, x, y = map(int, m.groups())
+
+            if 0 <= z < 16 and \
+               0 <= x < (1 << z) and \
+               0 <= y < (1 << z):
+                self.send_tile(z, x, y)
+                return
+
+        self.error_not_found()
+
+    def error_not_found(self):
+        self.send_response(requests.codes.not_found)
+
+    def send_tile(self, z, x, y):
+        from requests_futures.sessions import FuturesSession
+
+        session = FuturesSession()
+
+        futures = {}
+        tile_map = self.server.renderer.tiles_for(z, x, y)
+        for name, coord in tile_map.iteritems():
+            z, x, y = coord
+            url = self.server.url_pattern \
+                             .replace("{z}", str(z)) \
+                             .replace("{x}", str(x)) \
+                             .replace("{y}", str(y))
+
+            futures[name] = session.get(url)
+
+        tiles = {}
+        for name, fut in futures.iteritems():
+            res = fut.result()
+
+            if res.status_code != requests.codes.ok:
+                self.send_response(res.status_code)
+                return
+
+            tiles[name] = Tile(res.content)
+
+        im = self.server.renderer.render(tiles)
 
         self.send_response(200)
+        self.send_header('Cache-control', 'max-age=300')
         self.end_headers()
 
         im.save(self.wfile, 'PNG')
@@ -109,13 +184,14 @@ def send_template(self, template_name):
 
 
 class ThreadedHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
-    def __init__(self, server_address, handler_class, url_pattern):
+    def __init__(self, server_address, handler_class, url_pattern, renderer):
         http.server.HTTPServer.__init__(self, server_address, handler_class)
         self.url_pattern = url_pattern
+        self.renderer = renderer
 
 
-def serve_http(url, port):
-    httpd = ThreadedHTTPServer(("", port), Handler, url)
+def serve_http(url, port, renderer):
+    httpd = ThreadedHTTPServer(("", port), Handler, url, renderer)
     print("Listening on port %d. Point your browser towards "
           "http://localhost:%d/" % (port, port))
     httpd.serve_forever()

setup.py

@@ -32,6 +32,7 @@
         'click',
         'PIL',
         'requests',
+        'requests_futures',
         'squarify',
     ],
     entry_points=dict(