From 32a1866cbc60c82cb4940d7c2d06825661c82413 Mon Sep 17 00:00:00 2001 From: Emilio Mariscal Date: Mon, 18 Nov 2024 14:13:33 -0300 Subject: [PATCH] Black tests --- API/api_worker.py | 33 +++++++++----- src/app.py | 2 - src/post_processing/geojson_stats.py | 1 + src/post_processing/processor.py | 10 ++--- src/post_processing/transliterator.py | 10 +++-- src/validation/models.py | 64 +++++++++++++-------------- 6 files changed, 65 insertions(+), 55 deletions(-) diff --git a/API/api_worker.py b/API/api_worker.py index 9a1262cc..92756c7b 100644 --- a/API/api_worker.py +++ b/API/api_worker.py @@ -224,16 +224,18 @@ def process_raw_data(self, params, user=None): geojson_stats_html = None download_html_url = None if "include_stats" or "include_translit" in params.dict(): - post_processor = PostProcessor({ - "include_stats": params.include_stats, - "include_translit": params.include_translit - }) + post_processor = PostProcessor( + { + "include_stats": params.include_stats, + "include_translit": params.include_translit, + } + ) if params.include_stats: post_processor.filters = params.filters post_processor.init() - + geom_area, geom_dump, working_dir = RawData( params, str(self.request.id) ).extract_current_data(file_parts, post_processor.post_process_line) @@ -244,16 +246,23 @@ def process_raw_data(self, params, user=None): # Create a HTML summary of stats if params.include_stats_html: tpl = "stats" - if 'waterway' in post_processor.geoJSONStats.config.keys: + if "waterway" in post_processor.geoJSONStats.config.keys: tpl = "stats_waterway" - elif 'highway' in post_processor.geoJSONStats.config.keys: + elif "highway" in post_processor.geoJSONStats.config.keys: tpl = "stats_highway" - elif 'building' in post_processor.geoJSONStats.config.keys: + elif "building" in post_processor.geoJSONStats.config.keys: tpl = "stats_building" project_root = pathlib.Path(__file__).resolve().parent - tpl_path = os.path.join(project_root, "../src/post_processing/{tpl}_tpl.html".format(tpl=tpl)) - geojson_stats_html = post_processor.geoJSONStats.html(tpl_path).build() - upload_html_path = os.path.join(working_dir, os.pardir, f"{exportname_parts[-1]}.html") + tpl_path = os.path.join( + project_root, + "../src/post_processing/{tpl}_tpl.html".format(tpl=tpl), + ) + geojson_stats_html = post_processor.geoJSONStats.html( + tpl_path + ).build() + upload_html_path = os.path.join( + working_dir, os.pardir, f"{exportname_parts[-1]}.html" + ) with open(upload_html_path, "w") as f: f.write(geojson_stats_html) @@ -291,7 +300,7 @@ def process_raw_data(self, params, user=None): upload_file_path = file_path inside_file_size += os.path.getsize(file_path) break # only take one file inside dir , if contains many it should be inside zip - + # check if download url will be generated from s3 or not from config if use_s3_to_upload: file_transfer_obj = S3FileTransfer() diff --git a/src/app.py b/src/app.py index 580530a5..293f5a81 100644 --- a/src/app.py +++ b/src/app.py @@ -2258,5 +2258,3 @@ def get_summary_stats(self, start_date, end_date, group_by): result = self.cur.fetchall() self.d_b.close_conn() return [dict(item) for item in result] - - diff --git a/src/post_processing/geojson_stats.py b/src/post_processing/geojson_stats.py index edf6bdf4..3935634a 100644 --- a/src/post_processing/geojson_stats.py +++ b/src/post_processing/geojson_stats.py @@ -4,6 +4,7 @@ CONFIG_AREA = ["building"] CONFIG_LENGTH = ["highway", "waterway"] + class GeoJSONStats(Stats): """Used for collecting stats while processing GeoJSON files line by line""" diff --git a/src/post_processing/processor.py b/src/post_processing/processor.py index 25ed726f..c7806d70 100644 --- a/src/post_processing/processor.py +++ b/src/post_processing/processor.py @@ -1,9 +1,9 @@ - import json from .transliterator import Transliterator from .geojson_stats import GeoJSONStats -class PostProcessor(): + +class PostProcessor: """Used for posst-process data while processing GeoJSON files line by line""" options = {} @@ -21,10 +21,10 @@ def post_process_line(self, line: str): line_object = json.loads(line) for fn in self.functions: - fn(line_object) + fn(line_object) return json.dumps(line_object) - + def init(self): """ Initialize post-processor @@ -36,4 +36,4 @@ def init(self): if self.options["include_translit"]: self.transliterator = Transliterator() - self.functions.append(self.transliterator.translit) \ No newline at end of file + self.functions.append(self.transliterator.translit) diff --git a/src/post_processing/transliterator.py b/src/post_processing/transliterator.py index 1644c241..e2ed695c 100644 --- a/src/post_processing/transliterator.py +++ b/src/post_processing/transliterator.py @@ -1,8 +1,9 @@ from transliterate import translit, get_available_language_codes -class Transliterator(): + +class Transliterator: """Used for transliterate names while processing GeoJSON files line by line""" - + def __init__(self): self.available_language_codes = get_available_language_codes() self.name_tags = [f"name:{x}" for x in self.available_language_codes] @@ -16,5 +17,6 @@ def translit(self, line): if tag in line["properties"]["tags"]: translit_tag = "{tag}-translit".format(tag=tag) if not translit_tag in line["properties"]["tags"]: - line["properties"]["tags"][translit_tag] = \ - translit(line["properties"]["tags"][tag], code, reversed=True) + line["properties"]["tags"][translit_tag] = translit( + line["properties"]["tags"][tag], code, reversed=True + ) diff --git a/src/validation/models.py b/src/validation/models.py index 14cf5b6f..f97f9100 100644 --- a/src/validation/models.py +++ b/src/validation/models.py @@ -311,22 +311,22 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin): max_length=3, example="NPL", ) - geometry: Optional[ - Union[Polygon, MultiPolygon, Feature, FeatureCollection] - ] = Field( - default=None, - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, + geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = ( + Field( + default=None, + example={ + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + ) ) @validator("geometry", pre=True, always=True) @@ -632,22 +632,22 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin): max_length=3, example="USA", ) - geometry: Optional[ - Union[Polygon, MultiPolygon, Feature, FeatureCollection] - ] = Field( - default=None, - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, + geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = ( + Field( + default=None, + example={ + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + ) ) @validator("geometry", pre=True, always=True)