diff --git a/reporter/animate/.DS_Store b/reporter/animate/.DS_Store new file mode 100644 index 0000000..836a4ab Binary files /dev/null and b/reporter/animate/.DS_Store differ diff --git a/reporter/animate/__init__.py b/reporter/animate/__init__.py new file mode 100644 index 0000000..cc4b827 --- /dev/null +++ b/reporter/animate/__init__.py @@ -0,0 +1,5 @@ +# coding=utf-8 +"""Init for test package. +:copyright: (c) 2018 by Pierre-Alix Tremblay +:license: GPLv3, see LICENSE for more details. +""" diff --git a/reporter/animate/bin/encode b/reporter/animate/bin/encode new file mode 100755 index 0000000..8953451 Binary files /dev/null and b/reporter/animate/bin/encode differ diff --git a/reporter/animate/bin/render b/reporter/animate/bin/render new file mode 100755 index 0000000..25c02ee Binary files /dev/null and b/reporter/animate/bin/render differ diff --git a/reporter/animate/bin/snap b/reporter/animate/bin/snap new file mode 100755 index 0000000..b36809d Binary files /dev/null and b/reporter/animate/bin/snap differ diff --git a/reporter/animate/frame.py b/reporter/animate/frame.py new file mode 100644 index 0000000..189e3b7 --- /dev/null +++ b/reporter/animate/frame.py @@ -0,0 +1,184 @@ +import subprocess +import re +import os +from reporter import config +from reporter.animate.pathor import Pathor +# from reporter.utilities import LOGGER + + +class Frame: + + """ + Frame is an object representing a frame of the final GIF + A frame has an ID (a date). + A frame gathers all ways that have the same date. + """ + def __init__(self, frame_id): + """ + Constructor + + :param frame_id: the id of the frame with the format: + YYYY-MM (e.g: 2010-01) + :type frame_id: string + """ + self.frame_id = frame_id + self.paths = Pathor(self.frame_id) + + def build(self, osm_data): + """ + Build a frame.png (image.png + label.png) from a frame_id: + - Fetch the coordinates that belongs to the frame + - encode and render the image + - render the label + - render the frame + + :param osm_data: Data that every frame must access: ways and bounds + :type osm_data: OsmData instance + + """ + self.osm_data = osm_data + coordinates = self.osm_data.filter_coordinates_for_frame(self.frame_id) + + with open(self.paths.ways, 'w') as frame: + frame.write(''.join(coordinates)) + + self.encode_image() + self.render_image() + self.resize_image() + self.render_label() + self.render_frame() + + def width(self): + """ + Run the command `identify` on the image.png + + :returns: the width of the image.png + :rtype: string + """ + image_output = subprocess.check_output(['identify', self.paths.image]) + return re.search('PNG (\\d+)x\\d+', str(image_output)).group(1) + + def height(self): + """ + Run the command `identify` on the image.png + + :returns: the height of the image.png + :rtype: string + """ + image_output = subprocess.check_output(['identify', self.paths.image]) + return re.search('PNG \\d+x(\\d+)', str(image_output)).group(1) + + def encode_image(self): + """ + Encode coordinates + + :returns: None + """ + if os.path.exists(self.paths.ways): + command = 'cat {} | {}encode -o {} -z {}'.format( + self.paths.ways, + config.BIN_PATH, + self.paths.encoded_frame, + config.ZOOM_LEVEL) + + os.system(command) + + def render_image(self): + """ + Render an image with the encoded frame and the boundaries + + :returns: None + """ + if (os.path.exists(self.paths.encoded_frame) and + self.osm_data.bounds is not None): + command1 = '{}render -t 0 -A -- "{}" {}'.format( + config.BIN_PATH, + self.paths.encoded_frame, + config.ZOOM_LEVEL) + + command2 = '{} {} {} {}'.format( + self.osm_data.bounds['minlat'], + self.osm_data.bounds['minlon'], + self.osm_data.bounds['maxlat'], + self.osm_data.bounds['maxlon']) + + command3 = '> {}'.format(self.paths.image) + + os.system('{} {} {}'.format(command1, command2, command3)) + + def resize_image(self): + command = 'convert {} -resize 940x350 {}'.format( + self.paths.image, self.paths.image) + os.system(command) + + def render_label(self): + """ + Create a .png file of the label + + :returns: None + """ + command1 = 'convert -size {}x50 -gravity Center' \ + ' -background black'.format(self.width()) + command2 = '-stroke white -fill white label:\'{}\' {}'.format( + self.frame_id, self.paths.label) + + os.system('{} {}'.format(command1, command2)) + + def render_frame(self): + """ + Render a frame using the image and the label + + :returns: None + """ + command = 'convert -append {} {} {}'.format( + self.paths.image, + self.paths.label, + self.paths.frame) + + os.system(command) + + +def build_init_frame(): + """ Build the init frame. + We need to build the first frame before building the init frame. + The init frame is an image with black background. + It has the same height/width as the first frame + """ + + # get the first frame directory + first_frame_id = os.listdir(config.TMP_PATH)[0] + # create a instance of this Frame + first_frame = Frame(first_frame_id) + + # create the directory for the init frame: 0000-00 + init_frame_dir = '{}/0000-00'.format(config.TMP_PATH) + os.makedirs(init_frame_dir) + + frame = '{}/frame.png'.format(init_frame_dir) + + # create the 0000-00/frame.png + command = 'convert -size {}x{} canvas:black {}'.format( + first_frame.width(), + str(int(first_frame.height()) + 50), + frame) + + os.system(command) + + +def build_gif(gif_file): + """ Build the final Gif + + :param gif_file: the path of the final GIF + :type gif_file: string + + :returns: None + """ + frames = '{}*/frame.png'.format(config.TMP_PATH) + + command = 'convert -coalesce -dispose 1 -delay 20 -loop 0 {} {}'.format( + frames, gif_file) + os.system(command) + + command = 'convert {} \\( +clone -set delay 500 \\)' \ + ' +swap +delete {}'.format(gif_file, gif_file) + os.system(command) diff --git a/reporter/animate/osm_data.py b/reporter/animate/osm_data.py new file mode 100644 index 0000000..a42ecc1 --- /dev/null +++ b/reporter/animate/osm_data.py @@ -0,0 +1,110 @@ +import re +import os +from bs4 import BeautifulSoup +from reporter.animate.util import ( + format_timestamp, + mkdir_tmp +) +# from reporter.utilities import LOGGER +from reporter import config + + +class OsmData: + """ OsmData object to store data from the OSM file: + - tags + - tag + - datamap file content (from the snap command) + Those data will be used by every Frame to be built. + """ + def __init__(self, osm_file): + """ Constructor + + :param osm_file: The path of the OSM file + :type osm_file: str + + :returns: A OsmData instance + :rtype: OsmData + """ + try: + soup = BeautifulSoup(open(osm_file), 'html.parser') + except FileNotFoundError: + self.ways = None + self.bounds = None + self.datamap = None + else: + self.ways = soup.find_all('way') + self.bounds = soup.find('bounds') + self.datamap = snap_datamap(osm_file) + + def frames(self): + """ + Create an array of unique date (YYYY-MM) + + :returns: A sorted set/map of unique date (frame) + :rtype: set + """ + return sorted(set(map( + lambda way: + format_timestamp(way['timestamp']), self.ways + ))) + + def filter_coordinates_for_frame(self, frame): + """ + Find coordinates that belongs to a frame + + :param frame: the id of the frame (e.g: '2010-01') + :type frame: str + + :returns: A list of coordinates + :rtype: list + """ + ways = self.map_ways_for_frame(frame) + return list(filter( + lambda line: + re.search("id=(\\d+)", line).group(1) in ways, self.datamap + )) + + def map_ways_for_frame(self, frame): + """ + Map a list containing only the id of the Way objects + + :param frame: the id of the frame (e.g: '2010-01') + :type frame: str + + :returns: A list of way id (str) + :rtype: list + """ + return list(map( + lambda way: way['id'], self._filter_ways_for_frame(frame) + )) + + def _filter_ways_for_frame(self, frame): + """ + Filter Way objects that belongs to a frame + + :param frame: the id of the frame (e.g: '2010-01') + :type frame: str + + :returns: A list of Way objects + :rtype: list + """ + return list(filter( + lambda way: + frame == format_timestamp(way['timestamp']), self.ways + )) + + +def snap_datamap(osm_file): + mkdir_tmp() + if os.path.exists(osm_file): + command = 'cat {} | {}snap > {}'.format( + osm_file, + config.BIN_PATH, + config.DATAMAP) + + os.system(command) + try: + with open(config.DATAMAP, 'r') as datamapfile: + return datamapfile.readlines() + except FileNotFoundError: + return None diff --git a/reporter/animate/osm_gif.py b/reporter/animate/osm_gif.py new file mode 100644 index 0000000..a27f72f --- /dev/null +++ b/reporter/animate/osm_gif.py @@ -0,0 +1,66 @@ +import os +from reporter import config +from reporter.animate.osm_data import OsmData +from reporter.animate.frame import ( + Frame, + build_init_frame, + build_gif +) +# from reporter.utilities import LOGGER + + +class OsmGif: + """ + OsmGif transforms an OSM file into a GIF file + """ + + def __init__(self, osm_file): + """ + Constructor + + :param osm_file: the path of an OSM file + :type osm_file: string + + :returns: an instance of OsmGif + :rtype: OsmGif + """ + self.osm_file = osm_file + self.filename = os.path.splitext(os.path.basename(self.osm_file))[0] + self.gif_file = '{}{}.gif'.format(config.GIF_PATH, self.filename) + + def run(self): + """ + Where the magic happens. + """ + + # if the gif already exists, we return immediately + if os.path.isfile(self.gif_file): + return + + # get an OsmData object + osm_data = OsmData(self.osm_file) + + # build each frame + for frame_id in osm_data.frames(): + Frame(frame_id).build(osm_data) + + # build the init frame + build_init_frame() + + # build the final gif + build_gif(self.gif_file) + + +def osm_to_gif(osm_file): + """ + Animate an OSM file: create an object OsmGif and run + + :param osm_file: path of an OSM file + :type osm_file: string + + :returns: path of the gif file + :rtype: string + """ + osm_gif = OsmGif(osm_file) + osm_gif.run() + return osm_gif.gif_file diff --git a/reporter/animate/pathor.py b/reporter/animate/pathor.py new file mode 100644 index 0000000..1ca8c5a --- /dev/null +++ b/reporter/animate/pathor.py @@ -0,0 +1,31 @@ +import os +from reporter import config + + +class Pathor: + """ + Pathor Object + Each frame has a Pathor object to store path of the frame's files: + - ways.txt + - image.png + - frame.png + - label.png + - encoded_frame/ + """ + def __init__(self, frame_id): + """ + Constructor + :param frame_id: the id of a frame (e.g: 2010-01) + :type frame_id: string + """ + self.frame_id = frame_id + self.path = '{}{}/'.format(config.TMP_PATH, frame_id) + + if not os.path.exists(self.path): + os.makedirs(self.path) + + self.ways = '{}ways.txt'.format(self.path) + self.image = '{}image.png'.format(self.path) + self.frame = '{}frame.png'.format(self.path) + self.label = '{}label.png'.format(self.path) + self.encoded_frame = '{}encoded_frame'.format(self.path) diff --git a/reporter/animate/util.py b/reporter/animate/util.py new file mode 100644 index 0000000..d22b1f5 --- /dev/null +++ b/reporter/animate/util.py @@ -0,0 +1,16 @@ +import os +import shutil +from dateutil.parser import parse +from reporter import config +# from reporter.utilities import LOGGER +# This class must be refactored and tested + + +def mkdir_tmp(): + if os.path.exists(config.TMP_PATH): + shutil.rmtree(config.TMP_PATH) + os.makedirs(config.TMP_PATH) + + +def format_timestamp(timestamp): + return parse(str(timestamp)).strftime("%Y-%m") diff --git a/reporter/config/default.py b/reporter/config/default.py index b382b01..23b8799 100644 --- a/reporter/config/default.py +++ b/reporter/config/default.py @@ -27,3 +27,17 @@ # Options for the osm2pgsql command line OSM2PGSQL_OPTIONS = os.environ.get('OSM2PGSQL_OPTIONS') \ if os.environ.get('OSM2PGSQL_OPTIONS', False) else '' + +# OSMGIF env variables +# GIF_DIR = os.path.dirname(os.path.abspath(__file__)) +# where to store the gif so it's accessible trough a web browser +GIF_PATH = os.path.dirname( + os.path.realpath(__file__)) + '/../static/gif/' +# zoom +ZOOM_LEVEL = '15' # 15 is big +# working directory +TMP_PATH = CACHE_DIR + '/osm-gif/' +# datamap file +DATAMAP = TMP_PATH + "datamapfile" +# executables directory +BIN_PATH = os.path.dirname(os.path.realpath(__file__)) + '/../animate/bin/' diff --git a/reporter/queries.py b/reporter/queries.py index d4d09bd..fcb8b2f 100644 --- a/reporter/queries.py +++ b/reporter/queries.py @@ -91,6 +91,7 @@ ) EVACUATION_CENTERS_OVERPASS_QUERY = ( + '[bbox: {SW_lat},{SW_lng},{NE_lat},{NE_lng}];' '(' 'node["evacuation_center"="yes"]' '({SW_lat},{SW_lng},{NE_lat},{NE_lng});' @@ -103,6 +104,7 @@ 'out {print_mode};') BUILDINGS_OVERPASS_QUERY = ( + '[bbox: {SW_lat},{SW_lng},{NE_lat},{NE_lng}];' '(' 'node["building"]' '({SW_lat},{SW_lng},{NE_lat},{NE_lng});' @@ -115,6 +117,7 @@ 'out {print_mode};') ROADS_OVERPASS_QUERY = ( + '[bbox: {SW_lat},{SW_lng},{NE_lat},{NE_lng}];' '(' 'node["highway"]' '({SW_lat},{SW_lng},{NE_lat},{NE_lng});' @@ -127,6 +130,7 @@ 'out {print_mode};') FLOOD_PRONE_OVERPASS_QUERY = ( + '[bbox: {SW_lat},{SW_lng},{NE_lat},{NE_lng}];' '(' 'way["flood_prone"="yes"]' '({SW_lat},{SW_lng},{NE_lat},{NE_lng});' diff --git a/reporter/static/img/loader.gif b/reporter/static/img/loader.gif new file mode 100755 index 0000000..64e3c8c Binary files /dev/null and b/reporter/static/img/loader.gif differ diff --git a/reporter/templates/base.html b/reporter/templates/base.html index b42bcf5..2c478ec 100644 --- a/reporter/templates/base.html +++ b/reporter/templates/base.html @@ -55,12 +55,24 @@

OpenStreetMap Reporter

-
+
+
+ +
+ + +
-
+

Filters

@@ -212,7 +224,38 @@

{{ current_tag_name|capitalize|replace('_', ' ')}} Contr clickable: false }).addTo(map); window.bbox = map.getBounds().toBBoxString(); + + $('#animate').click(function() { + var current_bbox = map.getBounds().toBBoxString(); + + if (current_bbox === window.bbox) { + $('#gif').show(); + $('#map').hide(); + $('#animate').hide(); + $('#loading').show(); + $.ajax({ + url: 'animate', + data: { + filename: '{{ filename }}' + } + }).done(function(result) { + source = "/static/gif/" + result["result"]; + $('#loading').hide(); + $('#backtomap').show(); + $('#gif').html('') + }); + } else { + alert("You must refresh first."); + } + }); + $('#backtomap').click(function() { + $('#gif').hide(); + $('#map').show(); + $('#animate').show(); + $('#backtomap').hide(); + }); }); + $(function () { {# Pie chart for all users #} var myData = [ diff --git a/reporter/test/helpers.py b/reporter/test/helpers.py index 26706f0..4fecac1 100644 --- a/reporter/test/helpers.py +++ b/reporter/test/helpers.py @@ -11,3 +11,40 @@ 'test_data', 'swellendam.osm' ) + +UNKNOWN_OSMFILE_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'test_data', + 'unknown_osm_file.osm' +) + +SWELLENDAM_OSMFILE_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'test_data', + 'swellendam_with_bounds.osm' +) + +WAYS_FILE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'test_data', + 'ways.txt') + +BIG_IMAGE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'test_data', + 'big_image.png') + +TMP_BIG_IMAGE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'test_data', + 'tmp_big_image.png') + +SMALL_IMAGE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'test_data', + 'small_image.png') + +TMP_SMALL_IMAGE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'test_data', + 'tmp_small_image.png') diff --git a/reporter/test/test_data/big_image.png b/reporter/test/test_data/big_image.png new file mode 100644 index 0000000..74bd415 Binary files /dev/null and b/reporter/test/test_data/big_image.png differ diff --git a/reporter/test/test_data/small_image.png b/reporter/test/test_data/small_image.png new file mode 100644 index 0000000..05d74d8 Binary files /dev/null and b/reporter/test/test_data/small_image.png differ diff --git a/reporter/test/test_data/swellendam_with_bounds.osm b/reporter/test/test_data/swellendam_with_bounds.osm new file mode 100644 index 0000000..8462a9a --- /dev/null +++ b/reporter/test/test_data/swellendam_with_bounds.osm @@ -0,0 +1,6539 @@ + + +The data included in this document is from www.openstreetmap.org. The data is made available under ODbL. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/reporter/test/test_data/ways.txt b/reporter/test/test_data/ways.txt new file mode 100644 index 0000000..488518a --- /dev/null +++ b/reporter/test/test_data/ways.txt @@ -0,0 +1,9 @@ +-34.029976,20.431830 -34.030593,20.432838 -34.030990,20.433494 // id=47587914;highway=residential;name=Maynier Street +-34.028784,20.432593 -34.027742,20.430783 // id=47587919;highway=residential;name=Bethel Street +-34.018207,20.443325 -34.018842,20.442527 -34.019001,20.442345 -34.019263,20.441960 -34.020105,20.440892 -34.020820,20.439954 // id=47587923;highway=residential;name=Buitekant Street +-34.011420,20.458470 -34.010499,20.457060 // id=47587942;highway=residential +-34.010499,20.457060 -34.011010,20.456547 -34.011575,20.456000 -34.012073,20.455492 // id=47587943;highway=residential;name=Geelhout +-34.017246,20.442964 -34.017007,20.444048 // id=47884041;highway=residential +-34.017212,20.442132 -34.017943,20.441130 -34.018207,20.440339 // id=47884042;highway=residential;name=Kloof Street +-34.017131,20.452819 -34.017425,20.453216 -34.017865,20.452753 -34.017600,20.452352 // id=47884043;highway=residential +-34.016580,20.448608 -34.016100,20.448233 // id=47884044;highway=residential;name=Human Street diff --git a/reporter/test/test_frame.py b/reporter/test/test_frame.py new file mode 100644 index 0000000..a95fc5b --- /dev/null +++ b/reporter/test/test_frame.py @@ -0,0 +1,142 @@ +import os +from reporter.test.logged_unittest import LoggedTestCase +from reporter.test.helpers import ( + SWELLENDAM_OSMFILE_PATH, + BIG_IMAGE, + TMP_BIG_IMAGE, + SMALL_IMAGE, + TMP_SMALL_IMAGE +) +# from reporter.utilities import LOGGER +from reporter import config +from reporter.animate.frame import ( + Frame, + build_init_frame, + build_gif +) +from reporter.animate.osm_data import OsmData +from reporter.animate.util import ( + mkdir_tmp +) + + +class FrameTestCase(LoggedTestCase): + def setUp(self): + mkdir_tmp() + + def test_resize_image_when_big(self): + """ + it should resize a big image to 940x350 + """ + # new fake frame + frame = Frame('frame_id') + + # use a copy of a big image + command = 'cp {} {}'.format(BIG_IMAGE, TMP_BIG_IMAGE) + os.system(command) + + # change frame's image's path + frame.paths.image = TMP_BIG_IMAGE + + # it should be a big image + self.assertEqual(frame.width(), '3760') + + frame.resize_image() + + # it should be resized to 940 + self.assertEqual(frame.width(), '940') + + # remove the resized image + command = 'rm {}'.format(TMP_BIG_IMAGE) + os.system(command) + + def test_resize_image_when_small(self): + """ + it should resize a small image to 940x350 + """ + + # new fake frame + frame = Frame('frame_id') + + # use a copy of a small image + command = 'cp {} {}'.format(SMALL_IMAGE, TMP_SMALL_IMAGE) + os.system(command) + + frame.paths.image = TMP_SMALL_IMAGE + + self.assertEqual(frame.width(), '469') + + frame.resize_image() + + # 938, 940, same, same + self.assertEqual(frame.width(), '938') + + # remove the resized image + command = 'rm {}'.format(TMP_SMALL_IMAGE) + os.system(command) + + def test_build_one_frame(self): + """ + it should build one frame: + """ + frame_id = '2010-01' + osm_data = OsmData(SWELLENDAM_OSMFILE_PATH) + working_dir = '{}{}'.format(config.TMP_PATH, frame_id) + + frame = Frame(frame_id) + frame.build(osm_data) + # - create a working directory for the frame + self.assertTrue(os.path.exists(working_dir)) + + # - write coordinates in dir/ways.txt + self.assertTrue(os.path.exists('{}/ways.txt'.format(working_dir))) + + # - create a working directory for the encoded frame + self.assertTrue( + os.path.exists('{}/encoded_frame/'.format(working_dir))) + + # - render the image + self.assertTrue(os.path.exists('{}/image.png'.format(working_dir))) + + # - render the label + self.assertTrue(os.path.exists('{}/label.png'.format(working_dir))) + + # - render the frame + self.assertTrue(os.path.exists('{}/frame.png'.format(working_dir))) + + def test_build_first_frame(self): + """ + it should build the frame 0000-00 + """ + + # first we build the first frame + frame_id = '2010-01' + osm_data = OsmData(SWELLENDAM_OSMFILE_PATH) + frame = Frame(frame_id) + frame.build(osm_data) + + init_frame_dir = '{}/0000-00'.format(config.TMP_PATH) + init_frame = '{}/frame.png'.format(init_frame_dir) + + # then we build the init frame + build_init_frame() + + self.assertTrue(os.path.exists(init_frame_dir)) + self.assertTrue(os.path.exists(init_frame)) + + def test_build_gif(self): + """ + it should build a gif of two frames (init + first_frame) + """ + + frame_id = '2010-01' + osm_data = OsmData(SWELLENDAM_OSMFILE_PATH) + frame = Frame(frame_id) + frame.build(osm_data) + build_init_frame() + + gif_file = '{}{}.gif'.format(config.GIF_PATH, 'my_gif') + + build_gif(gif_file) + + self.assertTrue(os.path.exists(gif_file)) diff --git a/reporter/test/test_osm_data.py b/reporter/test/test_osm_data.py new file mode 100644 index 0000000..f093f38 --- /dev/null +++ b/reporter/test/test_osm_data.py @@ -0,0 +1,119 @@ +import os +from reporter.test.logged_unittest import LoggedTestCase +from reporter.test.helpers import ( + UNKNOWN_OSMFILE_PATH, + SWELLENDAM_OSMFILE_PATH +) +from reporter.animate.osm_data import ( + OsmData, + snap_datamap +) +# from reporter.utilities import LOGGER +from reporter import config + + +class OsmDataTestCase(LoggedTestCase): + + def test_with_unknown_osm_file(self): + """ + When the OSM file does not exist + It should assigns None to + ways, bounds and datamap + """ + osm_file = UNKNOWN_OSMFILE_PATH + data = OsmData(osm_file) + self.assertEqual(data.ways, None) + self.assertEqual(data.bounds, None) + self.assertEqual(data.datamap, None) + + def test_with_known_osm_file_with_bounds(self): + """ + When the OSM file exists + It should parse the OSM file + to retrieve an array of tags + and a map of the tag. + """ + osm_file = SWELLENDAM_OSMFILE_PATH + data = OsmData(osm_file) + expected_way_count = 333 + expected_bounds = { + "minlat": "-34.0537260", + "minlon": "20.4114820", + "maxlat": "-34.0094830", + "maxlon": "20.4673580" + } + self.assertEqual(len(data.ways), expected_way_count) + self.assertEqual(data.bounds["minlat"], expected_bounds["minlat"]) + self.assertEqual(data.bounds["minlon"], expected_bounds["minlon"]) + self.assertEqual(data.bounds["maxlat"], expected_bounds["maxlat"]) + self.assertEqual(data.bounds["maxlon"], expected_bounds["maxlon"]) + + def test_map_ways_for_frame(self): + """ + It should find 9 ids for the frame '2010-01' + """ + osm_file = SWELLENDAM_OSMFILE_PATH + data = OsmData(osm_file) + + frame = '2010-01' + ways = data.map_ways_for_frame(frame) + self.assertEqual(len(ways), 9) + self.assertEqual(ways[0], '47587914') + + def test_filter_coordinates(self): + """ + It should find 9 coordinates for the frame '2010-01' + """ + osm_file = SWELLENDAM_OSMFILE_PATH + data = OsmData(osm_file) + + frame = '2010-01' + coordinates = data.filter_coordinates_for_frame(frame) + expected_coordinates = '-34.029976,20.431830 -34.030593,20.432838' + \ + ' -34.030990,20.433494 // id=47587914;highway=residential;' + \ + 'name=Maynier Street\n' + self.assertEqual(len(coordinates), 9) + self.assertEqual(coordinates[0], expected_coordinates) + + def test_frames(self): + """ + It should return 32 frames from this OSM file + """ + osm_file = SWELLENDAM_OSMFILE_PATH + data = OsmData(osm_file) + + frames = data.frames() + self.assertEqual(frames[0], '2010-01') + self.assertEqual(len(frames), 32) + + def test_snap_datamap_from_known_osm_file(self): + """ + It should snap an OSM file + """ + osm_file = SWELLENDAM_OSMFILE_PATH + datamap = snap_datamap(osm_file) + + # check the datamap file exists + self.assertTrue(os.path.exists(config.DATAMAP)) + + # check the datamap file's first line + expected_datamap = '-34.026440,20.448198 -34.026266,20.448653 ' + \ + '-34.025996,20.449497 -34.025467,20.451333' + \ + ' -34.024931,20.453213 -34.024680,20.454165' + \ + ' -34.024562,20.454708 -34.024493,20.455156 ' + \ + '-34.024379,20.456064 -34.024148,20.458149' + \ + ' // id=4361694;highway=trunk;oneway=no;ref=N2\n' + self.assertEqual(datamap[0], expected_datamap) + + def test_snap_datamap_from_unkwnwn_osm_file(self): + """ + It should return None as the osm file does not exist + """ + osm_file = UNKNOWN_OSMFILE_PATH + datamap = snap_datamap(osm_file) + + # check that the datamap file does not exist + self.assertFalse(os.path.exists(config.DATAMAP)) + + # check that datamap returns None + self.assertIsNone(datamap) diff --git a/reporter/test/test_osm_gif.py b/reporter/test/test_osm_gif.py new file mode 100644 index 0000000..1dd4ef2 --- /dev/null +++ b/reporter/test/test_osm_gif.py @@ -0,0 +1,51 @@ +import os +from reporter.test.logged_unittest import LoggedTestCase +from reporter.test.helpers import ( + SWELLENDAM_OSMFILE_PATH +) +from reporter.animate.osm_gif import ( + OsmGif, + osm_to_gif +) +# from reporter.utilities import LOGGER +from reporter import config + + +class OsmGifTestCase(LoggedTestCase): + def test_init(self): + """ + check that the osm filename is correct + check that the gif file is correct + """ + osm_file = SWELLENDAM_OSMFILE_PATH + osm_gif = OsmGif(osm_file) + self.assertEqual(osm_gif.filename, 'swellendam_with_bounds') + + gif_path = '{}swellendam_with_bounds.gif'.format(config.GIF_PATH) + self.assertEqual(osm_gif.gif_file, gif_path) + + def test_run(self): + """ + check that it creates a gif file + """ + osm_file = SWELLENDAM_OSMFILE_PATH + osm_gif = OsmGif(osm_file) + gif_path = '{}swellendam_with_bounds.gif'.format(config.GIF_PATH) + + if os.path.isfile(gif_path): + os.remove(gif_path) + + osm_gif.run() + + self.assertTrue(os.path.isfile(gif_path)) + + def test_osm_to_gif(self): + """ + check that it creates a gif file + """ + gif_path = '{}swellendam_with_bounds.gif'.format(config.GIF_PATH) + if os.path.isfile(gif_path): + os.remove(gif_path) + + gif_file = osm_to_gif(SWELLENDAM_OSMFILE_PATH) + self.assertEqual(gif_file, gif_path) diff --git a/reporter/test/test_pathor.py b/reporter/test/test_pathor.py new file mode 100644 index 0000000..cb515f4 --- /dev/null +++ b/reporter/test/test_pathor.py @@ -0,0 +1,18 @@ +from reporter.test.logged_unittest import LoggedTestCase +# from reporter.utilities import LOGGER +from reporter import config +from reporter.animate.pathor import Pathor + + +class PathorTestCase(LoggedTestCase): + def test_init(self): + frame_id = 'pathor-test' + working_dir = '{}{}/'.format(config.TMP_PATH, frame_id) + pathor = Pathor(frame_id) + + self.assertEqual(pathor.ways, '{}ways.txt'.format(working_dir)) + self.assertEqual(pathor.image, '{}image.png'.format(working_dir)) + self.assertEqual(pathor.frame, '{}frame.png'.format(working_dir)) + self.assertEqual(pathor.label, '{}label.png'.format(working_dir)) + self.assertEqual(pathor.encoded_frame, '{}encoded_frame'.format( + working_dir)) diff --git a/reporter/views.py b/reporter/views.py index f33b394..2518dfd 100644 --- a/reporter/views.py +++ b/reporter/views.py @@ -34,6 +34,10 @@ # noinspection PyPep8Naming from urllib.error import URLError +from reporter.animate.osm_gif import ( + osm_to_gif +) + @app.route('/') def home(): @@ -91,6 +95,11 @@ def home(): coordinates = dict((k, repr(v)) for k, v in coordinates.items()) download_url = '%s-shp' % TAG_MAPPING[tag_name] + if os.path.isfile(file_handle.name): + filename = os.path.basename(file_handle.name) + else: + filename = '' + context = dict( sorted_user_list=sorted_user_list, way_count=way_count, @@ -103,11 +112,22 @@ def home(): error=error, coordinates=coordinates, display_update_control=int(config.DISPLAY_UPDATE_CONTROL), + filename=filename ) # noinspection PyUnresolvedReferences return render_template('base.html', **context) +@app.route('/animate') +def animate_osm(): + + file_handle = request.args.get('filename', None) + tmp = '{}/{}'.format(config.CACHE_DIR, file_handle) + gif_file = osm_to_gif(tmp) + + return jsonify(result=os.path.basename(gif_file)) + + @app.route('/-shp') def download_feature(feature_type): """Generic request to download OSM data. diff --git a/requirements-dev.txt b/requirements-dev.txt index fd3f58e..ef212f2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,3 +5,7 @@ pep8 nosexcover nose selenium +bs4 +beautifulsoup4 +python-dateutil +pycodestyle \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index af95a74..c913e88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ flask raven cherrypy - +bs4 +beautifulsoup4 +python-dateutil