Skip to content

Commit

Permalink
Black tests
Browse files Browse the repository at this point in the history
  • Loading branch information
emi420 committed Nov 18, 2024
1 parent 7acb431 commit 32a1866
Show file tree
Hide file tree
Showing 6 changed files with 65 additions and 55 deletions.
33 changes: 21 additions & 12 deletions API/api_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,16 +224,18 @@ def process_raw_data(self, params, user=None):
geojson_stats_html = None
download_html_url = None
if "include_stats" or "include_translit" in params.dict():
post_processor = PostProcessor({
"include_stats": params.include_stats,
"include_translit": params.include_translit
})
post_processor = PostProcessor(
{
"include_stats": params.include_stats,
"include_translit": params.include_translit,
}
)

if params.include_stats:
post_processor.filters = params.filters

post_processor.init()

geom_area, geom_dump, working_dir = RawData(
params, str(self.request.id)
).extract_current_data(file_parts, post_processor.post_process_line)
Expand All @@ -244,16 +246,23 @@ def process_raw_data(self, params, user=None):
# Create a HTML summary of stats
if params.include_stats_html:
tpl = "stats"
if 'waterway' in post_processor.geoJSONStats.config.keys:
if "waterway" in post_processor.geoJSONStats.config.keys:
tpl = "stats_waterway"
elif 'highway' in post_processor.geoJSONStats.config.keys:
elif "highway" in post_processor.geoJSONStats.config.keys:
tpl = "stats_highway"
elif 'building' in post_processor.geoJSONStats.config.keys:
elif "building" in post_processor.geoJSONStats.config.keys:
tpl = "stats_building"
project_root = pathlib.Path(__file__).resolve().parent
tpl_path = os.path.join(project_root, "../src/post_processing/{tpl}_tpl.html".format(tpl=tpl))
geojson_stats_html = post_processor.geoJSONStats.html(tpl_path).build()
upload_html_path = os.path.join(working_dir, os.pardir, f"{exportname_parts[-1]}.html")
tpl_path = os.path.join(
project_root,
"../src/post_processing/{tpl}_tpl.html".format(tpl=tpl),
)
geojson_stats_html = post_processor.geoJSONStats.html(
tpl_path
).build()
upload_html_path = os.path.join(
working_dir, os.pardir, f"{exportname_parts[-1]}.html"
)
with open(upload_html_path, "w") as f:
f.write(geojson_stats_html)

Expand Down Expand Up @@ -291,7 +300,7 @@ def process_raw_data(self, params, user=None):
upload_file_path = file_path
inside_file_size += os.path.getsize(file_path)
break # only take one file inside dir , if contains many it should be inside zip

# check if download url will be generated from s3 or not from config
if use_s3_to_upload:
file_transfer_obj = S3FileTransfer()
Expand Down
2 changes: 0 additions & 2 deletions src/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -2258,5 +2258,3 @@ def get_summary_stats(self, start_date, end_date, group_by):
result = self.cur.fetchall()
self.d_b.close_conn()
return [dict(item) for item in result]


1 change: 1 addition & 0 deletions src/post_processing/geojson_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
CONFIG_AREA = ["building"]
CONFIG_LENGTH = ["highway", "waterway"]


class GeoJSONStats(Stats):
"""Used for collecting stats while processing GeoJSON files line by line"""

Expand Down
10 changes: 5 additions & 5 deletions src/post_processing/processor.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

import json
from .transliterator import Transliterator
from .geojson_stats import GeoJSONStats

class PostProcessor():

class PostProcessor:
"""Used for posst-process data while processing GeoJSON files line by line"""

options = {}
Expand All @@ -21,10 +21,10 @@ def post_process_line(self, line: str):
line_object = json.loads(line)

for fn in self.functions:
fn(line_object)
fn(line_object)

return json.dumps(line_object)

def init(self):
"""
Initialize post-processor
Expand All @@ -36,4 +36,4 @@ def init(self):

if self.options["include_translit"]:
self.transliterator = Transliterator()
self.functions.append(self.transliterator.translit)
self.functions.append(self.transliterator.translit)
10 changes: 6 additions & 4 deletions src/post_processing/transliterator.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from transliterate import translit, get_available_language_codes

class Transliterator():

class Transliterator:
"""Used for transliterate names while processing GeoJSON files line by line"""

def __init__(self):
self.available_language_codes = get_available_language_codes()
self.name_tags = [f"name:{x}" for x in self.available_language_codes]
Expand All @@ -16,5 +17,6 @@ def translit(self, line):
if tag in line["properties"]["tags"]:
translit_tag = "{tag}-translit".format(tag=tag)
if not translit_tag in line["properties"]["tags"]:
line["properties"]["tags"][translit_tag] = \
translit(line["properties"]["tags"][tag], code, reversed=True)
line["properties"]["tags"][translit_tag] = translit(
line["properties"]["tags"][tag], code, reversed=True
)
64 changes: 32 additions & 32 deletions src/validation/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,22 +311,22 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin):
max_length=3,
example="NPL",
)
geometry: Optional[
Union[Polygon, MultiPolygon, Feature, FeatureCollection]
] = Field(
default=None,
example={
"type": "Polygon",
"coordinates": [
[
[83.96919250488281, 28.194446860487773],
[83.99751663208006, 28.194446860487773],
[83.99751663208006, 28.214869548073377],
[83.96919250488281, 28.214869548073377],
[83.96919250488281, 28.194446860487773],
]
],
},
geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = (
Field(
default=None,
example={
"type": "Polygon",
"coordinates": [
[
[83.96919250488281, 28.194446860487773],
[83.99751663208006, 28.194446860487773],
[83.99751663208006, 28.214869548073377],
[83.96919250488281, 28.214869548073377],
[83.96919250488281, 28.194446860487773],
]
],
},
)
)

@validator("geometry", pre=True, always=True)
Expand Down Expand Up @@ -632,22 +632,22 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin):
max_length=3,
example="USA",
)
geometry: Optional[
Union[Polygon, MultiPolygon, Feature, FeatureCollection]
] = Field(
default=None,
example={
"type": "Polygon",
"coordinates": [
[
[83.96919250488281, 28.194446860487773],
[83.99751663208006, 28.194446860487773],
[83.99751663208006, 28.214869548073377],
[83.96919250488281, 28.214869548073377],
[83.96919250488281, 28.194446860487773],
]
],
},
geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = (
Field(
default=None,
example={
"type": "Polygon",
"coordinates": [
[
[83.96919250488281, 28.194446860487773],
[83.99751663208006, 28.194446860487773],
[83.99751663208006, 28.214869548073377],
[83.96919250488281, 28.214869548073377],
[83.96919250488281, 28.194446860487773],
]
],
},
)
)

@validator("geometry", pre=True, always=True)
Expand Down

0 comments on commit 32a1866

Please sign in to comment.