From b7f5b5b4b0b8a9811c48df8ac61d645dd668dd39 Mon Sep 17 00:00:00 2001 From: Moritz Schott Date: Thu, 16 Nov 2023 12:16:51 +0100 Subject: [PATCH] fix: support different python objects in time parameter and be more picky about other list parameters --- ohsome/clients.py | 34 +++--- ohsome/helper.py | 71 +++++++---- .../test_client/test_bbox_numpy.yaml | 59 --------- .../test_check_time_parameter_datetime.yaml | 2 +- ...st_check_time_parameter_datetimeindex.yaml | 61 ---------- .../test_check_time_parameter_list.yaml | 2 +- .../test_check_time_parameter_series.yaml | 60 --------- .../test_end_timestamp_as_time_input.yaml | 2 +- .../test_format_bboxes_dataframe.yaml | 2 +- .../test_client/test_format_bboxes_list.yaml | 10 +- .../test_format_bcircles_dataframe.yaml | 2 +- .../test_format_bcircles_geodataframe.yaml | 2 +- .../test_format_bcircles_list.yaml | 8 +- .../test_client/test_format_bpolys.yaml | 2 +- .../test_post_with_endpoint_string.yaml | 4 +- .../test_client/test_user_agent.yaml | 2 +- .../test_exceptions/test_disable_logging.yaml | 4 +- .../test_exception_connection_reset.yaml | 2 +- .../test_exception_invalid_parameters.yaml | 4 +- .../test_exceptions/test_invalid_url.yaml | 2 +- .../test_exceptions/test_log_bpolys.yaml | 4 +- .../test_exceptions/test_timeout_error.yaml | 4 +- .../test_contributions_centroid.yaml | 2 +- .../test_contributions_latest.yaml | 2 +- .../test_elementsFullHistory_geometry.yaml | 2 +- .../test_response/test_elements_count.yaml | 2 +- .../test_elements_count_groupby_boundary.yaml | 2 +- ...ts_count_groupby_boundary_groupby_tag.yaml | 2 +- .../test_elements_count_groupby_key.yaml | 2 +- .../test_elements_count_groupby_tag.yaml | 2 +- .../test_elements_count_groupby_type.yaml | 2 +- .../test_elements_count_ratio.yaml | 2 +- ...elements_count_ratio_groupby_boundary.yaml | 2 +- .../test_response/test_elements_density.yaml | 2 +- .../test_response/test_elements_geometry.yaml | 2 +- .../test_empty_geodataframe.yaml | 2 +- .../test_response/test_multi_index_false.yaml | 2 +- .../test_not_implemented_query.yaml | 2 +- .../test_response/test_users_timestamp.yaml | 2 +- ohsome/test/test_client.py | 45 ------- ohsome/test/test_exceptions.py | 2 +- ohsome/test/test_helper.py | 114 +++++++++++++++++- 42 files changed, 215 insertions(+), 323 deletions(-) delete mode 100644 ohsome/test/cassettes/test_client/test_bbox_numpy.yaml delete mode 100644 ohsome/test/cassettes/test_client/test_check_time_parameter_datetimeindex.yaml delete mode 100644 ohsome/test/cassettes/test_client/test_check_time_parameter_series.yaml diff --git a/ohsome/clients.py b/ohsome/clients.py index 4b960e8..94c24e8 100644 --- a/ohsome/clients.py +++ b/ohsome/clients.py @@ -3,9 +3,13 @@ """OhsomeClient classes to build and handle requests to ohsome API""" import json +import os import urllib import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + from ohsome import OhsomeException, OhsomeResponse from ohsome.constants import ( DEFAULT_LOG_DIR, @@ -17,12 +21,9 @@ extract_error_message_from_invalid_json, format_boundary, format_time, - format_lists, + convert_arrays, + format_list_parameters, ) -import os -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry -import numpy as np class _OhsomeBaseClient: @@ -355,21 +356,16 @@ def _format_parameters(self, params): :param params: Parameters for request :return: """ - for i in params.keys(): - if isinstance(params[i], np.ndarray): - params[i] = list(params[i]) self._parameters = params.copy() - try: - format_boundary(self._parameters) - except OhsomeException as e: - raise OhsomeException( - message=str(e), - error_code=440, - params=self._parameters, - url=self._url, - ) - self._parameters = format_time(self._parameters) - self._parameters = format_lists(self._parameters) + + self._parameters = convert_arrays(self._parameters) + + self._parameters = format_boundary(self._parameters) + + if self._parameters.get("time") is not None: + self._parameters["time"] = format_time(self._parameters.get("time")) + + self._parameters = format_list_parameters(self._parameters) def _construct_resource_url(self, endpoint=None): """ diff --git a/ohsome/helper.py b/ohsome/helper.py index 5ee8e87..974524c 100644 --- a/ohsome/helper.py +++ b/ohsome/helper.py @@ -4,35 +4,52 @@ """Ohsome utility functions""" import datetime +import re import geopandas as gpd +import numpy as np import pandas as pd from ohsome import OhsomeException -import re +def convert_arrays(params: dict) -> dict: + """Convert arrays to lists. -def format_time(params: dict) -> dict: - """ - Formats the 'time' parameter given as string, list of dates or pandas.Series or pandas.DateTimeIndex - :param params: - :return: + params: the request parameters """ - if isinstance(params["time"], pd.DatetimeIndex): - params["time"] = params["time"].strftime("%Y-%m-%dT%H:%M:%S").tolist() - elif isinstance(params["time"], pd.Series): - params["time"] = params["time"].tolist() - elif isinstance((params["time"]), list): - if isinstance((params["time"])[0], datetime.datetime): - params["time"] = [x.isoformat() for x in params["time"]] - elif isinstance(params["time"], datetime.datetime): - params["time"] = params["time"].strftime("%Y-%m-%dT%H:%M:%S") + for i in params.keys(): + if isinstance(params[i], np.ndarray): + assert ( + params[i].ndim == 1 + ), f"Only one dimensional arrays are supported for parameter {i}" + params[i] = list(params[i]) return params -def format_boundary(params): +def format_time(time: any) -> str: + """ + Formats the 'time' parameter + :param time: + :return: + """ + if isinstance(time, str): + return time + if isinstance(time, datetime.datetime) or isinstance(time, datetime.date): + return time.isoformat() + elif isinstance(time, list): + return ",".join([format_time(t) for t in time]) + if isinstance(time, pd.DatetimeIndex) or isinstance(time, pd.Series): + return format_time(time.to_list()) + else: + raise ValueError( + f"The given time format {type(time)} is not supported. Feel free to open an issue in " + "the ohsome-py repository for a feature request." + ) + + +def format_boundary(params: dict) -> dict: """ Formats the boundary parameters 'bboxes', 'bcircles' and 'bpolys' :param params: @@ -46,10 +63,14 @@ def format_boundary(params): params["bcircles"] = format_bcircles(params["bcircles"]) else: raise OhsomeException( - message="No valid boundary parameter is given. Specify one of the parameters 'bboxes', 'bpolys' or 'bcircles'.", + message="No valid boundary parameter is given. Specify one of the parameters 'bboxes', 'bpolys' or " + "'bcircles'.", error_code=440, + params=params, ) + return params + def format_bcircles(bcircles): """ @@ -168,15 +189,13 @@ def format_bpolys(bpolys): return bpolys -def format_lists(parameters: dict) -> dict: - """ - Converts parameters of type list to strings using ',' as seperator. - :param parameters: request parameters - :return: the modified parameters - """ - for k, v in parameters.items(): - if isinstance(v, list): - parameters[k] = ",".join(v) +def format_list_parameters(parameters: dict) -> dict: + """Converts parameters of type list to strings using ',' as seperator.""" + list_parameters = ["groupByKeys", "groupByValues", "properties"] + for param in list_parameters: + if isinstance(parameters.get(param), list): + parameters[param] = ",".join(parameters[param]) + return parameters diff --git a/ohsome/test/cassettes/test_client/test_bbox_numpy.yaml b/ohsome/test/cassettes/test_client/test_bbox_numpy.yaml deleted file mode 100644 index b60f95b..0000000 --- a/ohsome/test/cassettes/test_client/test_bbox_numpy.yaml +++ /dev/null @@ -1,59 +0,0 @@ -interactions: -- request: - body: bboxes=8.67066%2C49.41423%2C8.68177%2C49.4204&time=2010-01-01&filter=amenity%3Drestaurant+and+type%3Anode - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '105' - Content-Type: - - application/x-www-form-urlencoded - user-agent: - - ohsome-py/0.2.0 - method: POST - uri: https://api.ohsome.org/v1/elements/count - response: - body: - string: "{\n \"attribution\" : {\n \"url\" : \"https://ohsome.org/copyrights\",\n - \ \"text\" : \"\xA9 OpenStreetMap contributors\"\n },\n \"apiVersion\" - : \"1.10.1\",\n \"result\" : [ {\n \"timestamp\" : \"2010-01-01T00:00:00Z\",\n - \ \"value\" : 0.0\n } ]\n}" - headers: - Access-Control-Allow-Credentials: - - 'true' - Access-Control-Allow-Headers: - - Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers,Authorization - Access-Control-Allow-Methods: - - POST, GET - Access-Control-Allow-Origin: - - '*' - Access-Control-Max-Age: - - '3600' - Cache-Control: - - no-transform, public, max-age=31556926 - Connection: - - Keep-Alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Wed, 15 Nov 2023 12:04:54 GMT - Keep-Alive: - - timeout=5, max=100 - Server: - - Apache - Strict-Transport-Security: - - max-age=63072000; includeSubdomains; - Transfer-Encoding: - - chunked - vary: - - accept-encoding - status: - code: 200 - message: '' -version: 1 diff --git a/ohsome/test/cassettes/test_client/test_check_time_parameter_datetime.yaml b/ohsome/test/cassettes/test_client/test_check_time_parameter_datetime.yaml index 27d039a..dbd3f50 100644 --- a/ohsome/test/cassettes/test_client/test_check_time_parameter_datetime.yaml +++ b/ohsome/test/cassettes/test_client/test_check_time_parameter_datetime.yaml @@ -43,7 +43,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:47 GMT + - Thu, 16 Nov 2023 11:12:17 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_check_time_parameter_datetimeindex.yaml b/ohsome/test/cassettes/test_client/test_check_time_parameter_datetimeindex.yaml deleted file mode 100644 index 5a420d7..0000000 --- a/ohsome/test/cassettes/test_client/test_check_time_parameter_datetimeindex.yaml +++ /dev/null @@ -1,61 +0,0 @@ -interactions: -- request: - body: bcircles=0%3A8.678770065307615%2C49.414435400453954%2C100%7C1%3A8.697137832641602%2C49.41007968889129%2C150&time=2018-01-01T00%3A00%3A00%2C2018-01-02T00%3A00%3A00%2C2018-01-03T00%3A00%3A00&filter=amenity%3Drestaurant+and+type%3Away - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '231' - Content-Type: - - application/x-www-form-urlencoded - user-agent: - - ohsome-py/0.2.0 - method: POST - uri: https://api.ohsome.org/v1/elements/count - response: - body: - string: "{\n \"attribution\" : {\n \"url\" : \"https://ohsome.org/copyrights\",\n - \ \"text\" : \"\xA9 OpenStreetMap contributors\"\n },\n \"apiVersion\" - : \"1.10.1\",\n \"result\" : [ {\n \"timestamp\" : \"2018-01-01T00:00:00Z\",\n - \ \"value\" : 0.0\n }, {\n \"timestamp\" : \"2018-01-02T00:00:00Z\",\n - \ \"value\" : 0.0\n }, {\n \"timestamp\" : \"2018-01-03T00:00:00Z\",\n - \ \"value\" : 0.0\n } ]\n}" - headers: - Access-Control-Allow-Credentials: - - 'true' - Access-Control-Allow-Headers: - - Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers,Authorization - Access-Control-Allow-Methods: - - POST, GET - Access-Control-Allow-Origin: - - '*' - Access-Control-Max-Age: - - '3600' - Cache-Control: - - no-transform, public, max-age=31556926 - Connection: - - Keep-Alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Wed, 15 Nov 2023 12:04:47 GMT - Keep-Alive: - - timeout=5, max=100 - Server: - - Apache - Strict-Transport-Security: - - max-age=63072000; includeSubdomains; - Transfer-Encoding: - - chunked - vary: - - accept-encoding - status: - code: 200 - message: '' -version: 1 diff --git a/ohsome/test/cassettes/test_client/test_check_time_parameter_list.yaml b/ohsome/test/cassettes/test_client/test_check_time_parameter_list.yaml index 0746284..300838f 100644 --- a/ohsome/test/cassettes/test_client/test_check_time_parameter_list.yaml +++ b/ohsome/test/cassettes/test_client/test_check_time_parameter_list.yaml @@ -44,7 +44,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:46 GMT + - Thu, 16 Nov 2023 11:12:16 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_check_time_parameter_series.yaml b/ohsome/test/cassettes/test_client/test_check_time_parameter_series.yaml deleted file mode 100644 index a99fe64..0000000 --- a/ohsome/test/cassettes/test_client/test_check_time_parameter_series.yaml +++ /dev/null @@ -1,60 +0,0 @@ -interactions: -- request: - body: bcircles=0%3A8.678770065307615%2C49.414435400453954%2C100%7C1%3A8.697137832641602%2C49.41007968889129%2C150&time=2018-01-01%2C2018-01-02&filter=amenity%3Drestaurant+and+type%3Away - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '179' - Content-Type: - - application/x-www-form-urlencoded - user-agent: - - ohsome-py/0.2.0 - method: POST - uri: https://api.ohsome.org/v1/elements/count - response: - body: - string: "{\n \"attribution\" : {\n \"url\" : \"https://ohsome.org/copyrights\",\n - \ \"text\" : \"\xA9 OpenStreetMap contributors\"\n },\n \"apiVersion\" - : \"1.10.1\",\n \"result\" : [ {\n \"timestamp\" : \"2018-01-01T00:00:00Z\",\n - \ \"value\" : 0.0\n }, {\n \"timestamp\" : \"2018-01-02T00:00:00Z\",\n - \ \"value\" : 0.0\n } ]\n}" - headers: - Access-Control-Allow-Credentials: - - 'true' - Access-Control-Allow-Headers: - - Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers,Authorization - Access-Control-Allow-Methods: - - POST, GET - Access-Control-Allow-Origin: - - '*' - Access-Control-Max-Age: - - '3600' - Cache-Control: - - no-transform, public, max-age=31556926 - Connection: - - Keep-Alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Wed, 15 Nov 2023 12:04:47 GMT - Keep-Alive: - - timeout=5, max=100 - Server: - - Apache - Strict-Transport-Security: - - max-age=63072000; includeSubdomains; - Transfer-Encoding: - - chunked - vary: - - accept-encoding - status: - code: 200 - message: '' -version: 1 diff --git a/ohsome/test/cassettes/test_client/test_end_timestamp_as_time_input.yaml b/ohsome/test/cassettes/test_client/test_end_timestamp_as_time_input.yaml index 3662799..17345b1 100644 --- a/ohsome/test/cassettes/test_client/test_end_timestamp_as_time_input.yaml +++ b/ohsome/test/cassettes/test_client/test_end_timestamp_as_time_input.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:48 GMT + - Thu, 16 Nov 2023 11:12:17 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_format_bboxes_dataframe.yaml b/ohsome/test/cassettes/test_client/test_format_bboxes_dataframe.yaml index cb1d0f0..f309d07 100644 --- a/ohsome/test/cassettes/test_client/test_format_bboxes_dataframe.yaml +++ b/ohsome/test/cassettes/test_client/test_format_bboxes_dataframe.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:51 GMT + - Thu, 16 Nov 2023 11:12:20 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_format_bboxes_list.yaml b/ohsome/test/cassettes/test_client/test_format_bboxes_list.yaml index 506ebcc..8538a8b 100644 --- a/ohsome/test/cassettes/test_client/test_format_bboxes_list.yaml +++ b/ohsome/test/cassettes/test_client/test_format_bboxes_list.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:53 GMT + - Thu, 16 Nov 2023 11:12:22 GMT Keep-Alive: - timeout=5, max=100 Server: @@ -99,7 +99,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:53 GMT + - Thu, 16 Nov 2023 11:12:22 GMT Keep-Alive: - timeout=5, max=100 Server: @@ -156,7 +156,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:53 GMT + - Thu, 16 Nov 2023 11:12:22 GMT Keep-Alive: - timeout=5, max=100 Server: @@ -213,7 +213,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:54 GMT + - Thu, 16 Nov 2023 11:12:23 GMT Keep-Alive: - timeout=5, max=100 Server: @@ -270,7 +270,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:54 GMT + - Thu, 16 Nov 2023 11:12:23 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_format_bcircles_dataframe.yaml b/ohsome/test/cassettes/test_client/test_format_bcircles_dataframe.yaml index 3c87c6b..f3bfb1e 100644 --- a/ohsome/test/cassettes/test_client/test_format_bcircles_dataframe.yaml +++ b/ohsome/test/cassettes/test_client/test_format_bcircles_dataframe.yaml @@ -45,7 +45,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:48 GMT + - Thu, 16 Nov 2023 11:12:18 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_format_bcircles_geodataframe.yaml b/ohsome/test/cassettes/test_client/test_format_bcircles_geodataframe.yaml index ce11019..6839301 100644 --- a/ohsome/test/cassettes/test_client/test_format_bcircles_geodataframe.yaml +++ b/ohsome/test/cassettes/test_client/test_format_bcircles_geodataframe.yaml @@ -51,7 +51,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:51 GMT + - Thu, 16 Nov 2023 11:12:20 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_format_bcircles_list.yaml b/ohsome/test/cassettes/test_client/test_format_bcircles_list.yaml index 06e27c1..fcb0a13 100644 --- a/ohsome/test/cassettes/test_client/test_format_bcircles_list.yaml +++ b/ohsome/test/cassettes/test_client/test_format_bcircles_list.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:48 GMT + - Thu, 16 Nov 2023 11:12:18 GMT Keep-Alive: - timeout=5, max=100 Server: @@ -102,7 +102,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:50 GMT + - Thu, 16 Nov 2023 11:12:18 GMT Keep-Alive: - timeout=5, max=100 Server: @@ -159,7 +159,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:50 GMT + - Thu, 16 Nov 2023 11:12:19 GMT Keep-Alive: - timeout=5, max=100 Server: @@ -216,7 +216,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:50 GMT + - Thu, 16 Nov 2023 11:12:19 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_format_bpolys.yaml b/ohsome/test/cassettes/test_client/test_format_bpolys.yaml index f0eeb15..1e34e1f 100644 --- a/ohsome/test/cassettes/test_client/test_format_bpolys.yaml +++ b/ohsome/test/cassettes/test_client/test_format_bpolys.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:51 GMT + - Thu, 16 Nov 2023 11:12:20 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_post_with_endpoint_string.yaml b/ohsome/test/cassettes/test_client/test_post_with_endpoint_string.yaml index b5ed3d8..abe4cb3 100644 --- a/ohsome/test/cassettes/test_client/test_post_with_endpoint_string.yaml +++ b/ohsome/test/cassettes/test_client/test_post_with_endpoint_string.yaml @@ -50,7 +50,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:04:56 GMT + - Thu, 16 Nov 2023 11:12:23 GMT Keep-Alive: - timeout=5, max=100 Server: @@ -115,7 +115,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:04:56 GMT + - Thu, 16 Nov 2023 11:12:25 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_client/test_user_agent.yaml b/ohsome/test/cassettes/test_client/test_user_agent.yaml index cccaabc..903bc1f 100644 --- a/ohsome/test/cassettes/test_client/test_user_agent.yaml +++ b/ohsome/test/cassettes/test_client/test_user_agent.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:46 GMT + - Thu, 16 Nov 2023 11:12:16 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_exceptions/test_disable_logging.yaml b/ohsome/test/cassettes/test_exceptions/test_disable_logging.yaml index 2ffba13..097da7c 100644 --- a/ohsome/test/cassettes/test_exceptions/test_disable_logging.yaml +++ b/ohsome/test/cassettes/test_exceptions/test_disable_logging.yaml @@ -18,7 +18,7 @@ interactions: uri: https://api.ohsome.org/v1/elements/geometry response: body: - string: "{\n \"timestamp\" : \"2023-11-15T12:04:56.832877092\",\n \"status\" + string: "{\n \"timestamp\" : \"2023-11-16T11:12:25.74208788\",\n \"status\" : 413,\n \"message\" : \"The given query is too large in respect to the given timeout. Please use a smaller region and/or coarser time period.\",\n \"requestUrl\" : \"https://api.ohsome.org/v1/elements/geometry\"\n}" @@ -44,7 +44,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:04:56 GMT + - Thu, 16 Nov 2023 11:12:25 GMT Server: - Apache Strict-Transport-Security: diff --git a/ohsome/test/cassettes/test_exceptions/test_exception_connection_reset.yaml b/ohsome/test/cassettes/test_exceptions/test_exception_connection_reset.yaml index efa2612..9c3ecc6 100644 --- a/ohsome/test/cassettes/test_exceptions/test_exception_connection_reset.yaml +++ b/ohsome/test/cassettes/test_exceptions/test_exception_connection_reset.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:03 GMT + - Thu, 16 Nov 2023 11:12:33 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_exceptions/test_exception_invalid_parameters.yaml b/ohsome/test/cassettes/test_exceptions/test_exception_invalid_parameters.yaml index 1849b59..179e8be 100644 --- a/ohsome/test/cassettes/test_exceptions/test_exception_invalid_parameters.yaml +++ b/ohsome/test/cassettes/test_exceptions/test_exception_invalid_parameters.yaml @@ -18,7 +18,7 @@ interactions: uri: https://api.ohsome.org/v1/elements/count/groupBy/tag response: body: - string: "{\n \"timestamp\" : \"2023-11-15T12:05:03.195938737\",\n \"status\" + string: "{\n \"timestamp\" : \"2023-11-16T11:12:33.156443534\",\n \"status\" : 400,\n \"message\" : \"You need to give one groupByKey parameter, if you want to use groupBy/tag.\",\n \"requestUrl\" : \"https://api.ohsome.org/v1/elements/count/groupBy/tag\"\n}" headers: @@ -41,7 +41,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:03 GMT + - Thu, 16 Nov 2023 11:12:33 GMT Server: - Apache Strict-Transport-Security: diff --git a/ohsome/test/cassettes/test_exceptions/test_invalid_url.yaml b/ohsome/test/cassettes/test_exceptions/test_invalid_url.yaml index 04b82fc..25790c4 100644 --- a/ohsome/test/cassettes/test_exceptions/test_invalid_url.yaml +++ b/ohsome/test/cassettes/test_exceptions/test_invalid_url.yaml @@ -38,7 +38,7 @@ interactions: Content-Type: - text/html Date: - - Wed, 15 Nov 2023 12:04:56 GMT + - Thu, 16 Nov 2023 11:12:25 GMT ETag: - '"25c-5c7180820e5fc"' Keep-Alive: diff --git a/ohsome/test/cassettes/test_exceptions/test_log_bpolys.yaml b/ohsome/test/cassettes/test_exceptions/test_log_bpolys.yaml index 3a8c711..bc37984 100644 --- a/ohsome/test/cassettes/test_exceptions/test_log_bpolys.yaml +++ b/ohsome/test/cassettes/test_exceptions/test_log_bpolys.yaml @@ -18,7 +18,7 @@ interactions: uri: https://api.ohsome.org/v1/elements/count response: body: - string: "{\n \"timestamp\" : \"2023-11-15T12:04:57.04867339\",\n \"status\" + string: "{\n \"timestamp\" : \"2023-11-16T11:12:26.959734902\",\n \"status\" : 413,\n \"message\" : \"The given query is too large in respect to the given timeout. Please use a smaller region and/or coarser time period.\",\n \"requestUrl\" : \"https://api.ohsome.org/v1/elements/count\"\n}" @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:04:57 GMT + - Thu, 16 Nov 2023 11:12:26 GMT Server: - Apache Strict-Transport-Security: diff --git a/ohsome/test/cassettes/test_exceptions/test_timeout_error.yaml b/ohsome/test/cassettes/test_exceptions/test_timeout_error.yaml index bfc5b71..b264560 100644 --- a/ohsome/test/cassettes/test_exceptions/test_timeout_error.yaml +++ b/ohsome/test/cassettes/test_exceptions/test_timeout_error.yaml @@ -18,7 +18,7 @@ interactions: uri: https://api.ohsome.org/v1/elements/geometry response: body: - string: "{\n \"timestamp\" : \"2023-11-15T12:04:56.617508005\",\n \"status\" + string: "{\n \"timestamp\" : \"2023-11-16T11:12:25.571444582\",\n \"status\" : 413,\n \"message\" : \"The given query is too large in respect to the given timeout. Please use a smaller region and/or coarser time period.\",\n \"requestUrl\" : \"https://api.ohsome.org/v1/elements/geometry\"\n}" @@ -44,7 +44,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:04:56 GMT + - Thu, 16 Nov 2023 11:12:25 GMT Server: - Apache Strict-Transport-Security: diff --git a/ohsome/test/cassettes/test_response/test_contributions_centroid.yaml b/ohsome/test/cassettes/test_response/test_contributions_centroid.yaml index 28e5aa7..498b7ce 100644 --- a/ohsome/test/cassettes/test_response/test_contributions_centroid.yaml +++ b/ohsome/test/cassettes/test_response/test_contributions_centroid.yaml @@ -46,7 +46,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:05:11 GMT + - Thu, 16 Nov 2023 11:12:42 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_contributions_latest.yaml b/ohsome/test/cassettes/test_response/test_contributions_latest.yaml index c789838..5aa029c 100644 --- a/ohsome/test/cassettes/test_response/test_contributions_latest.yaml +++ b/ohsome/test/cassettes/test_response/test_contributions_latest.yaml @@ -50,7 +50,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:05:11 GMT + - Thu, 16 Nov 2023 11:12:42 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elementsFullHistory_geometry.yaml b/ohsome/test/cassettes/test_response/test_elementsFullHistory_geometry.yaml index a743501..06b3a64 100644 --- a/ohsome/test/cassettes/test_response/test_elementsFullHistory_geometry.yaml +++ b/ohsome/test/cassettes/test_response/test_elementsFullHistory_geometry.yaml @@ -50,7 +50,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:05:10 GMT + - Thu, 16 Nov 2023 11:12:41 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_count.yaml b/ohsome/test/cassettes/test_response/test_elements_count.yaml index 6bc6a6c..ae4a580 100644 --- a/ohsome/test/cassettes/test_response/test_elements_count.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_count.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:03 GMT + - Thu, 16 Nov 2023 11:12:33 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_count_groupby_boundary.yaml b/ohsome/test/cassettes/test_response/test_elements_count_groupby_boundary.yaml index f636420..450c2a1 100644 --- a/ohsome/test/cassettes/test_response/test_elements_count_groupby_boundary.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_count_groupby_boundary.yaml @@ -45,7 +45,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:07 GMT + - Thu, 16 Nov 2023 11:12:37 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_count_groupby_boundary_groupby_tag.yaml b/ohsome/test/cassettes/test_response/test_elements_count_groupby_boundary_groupby_tag.yaml index 0f34909..10feda1 100644 --- a/ohsome/test/cassettes/test_response/test_elements_count_groupby_boundary_groupby_tag.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_count_groupby_boundary_groupby_tag.yaml @@ -45,7 +45,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:07 GMT + - Thu, 16 Nov 2023 11:12:38 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_count_groupby_key.yaml b/ohsome/test/cassettes/test_response/test_elements_count_groupby_key.yaml index d1d961c..984aed7 100644 --- a/ohsome/test/cassettes/test_response/test_elements_count_groupby_key.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_count_groupby_key.yaml @@ -50,7 +50,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:04 GMT + - Thu, 16 Nov 2023 11:12:34 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_count_groupby_tag.yaml b/ohsome/test/cassettes/test_response/test_elements_count_groupby_tag.yaml index 2940706..dfec525 100644 --- a/ohsome/test/cassettes/test_response/test_elements_count_groupby_tag.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_count_groupby_tag.yaml @@ -128,7 +128,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:06 GMT + - Thu, 16 Nov 2023 11:12:36 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_count_groupby_type.yaml b/ohsome/test/cassettes/test_response/test_elements_count_groupby_type.yaml index 0e0a9b4..daa3e60 100644 --- a/ohsome/test/cassettes/test_response/test_elements_count_groupby_type.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_count_groupby_type.yaml @@ -50,7 +50,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:07 GMT + - Thu, 16 Nov 2023 11:12:37 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_count_ratio.yaml b/ohsome/test/cassettes/test_response/test_elements_count_ratio.yaml index b8bf70a..559ae04 100644 --- a/ohsome/test/cassettes/test_response/test_elements_count_ratio.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_count_ratio.yaml @@ -43,7 +43,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:08 GMT + - Thu, 16 Nov 2023 11:12:38 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_count_ratio_groupby_boundary.yaml b/ohsome/test/cassettes/test_response/test_elements_count_ratio_groupby_boundary.yaml index 7ce64e6..c5dcf00 100644 --- a/ohsome/test/cassettes/test_response/test_elements_count_ratio_groupby_boundary.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_count_ratio_groupby_boundary.yaml @@ -50,7 +50,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:08 GMT + - Thu, 16 Nov 2023 11:12:39 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_density.yaml b/ohsome/test/cassettes/test_response/test_elements_density.yaml index b770753..53df48d 100644 --- a/ohsome/test/cassettes/test_response/test_elements_density.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_density.yaml @@ -42,7 +42,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:04 GMT + - Thu, 16 Nov 2023 11:12:34 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_elements_geometry.yaml b/ohsome/test/cassettes/test_response/test_elements_geometry.yaml index 0c5e72d..ac725b0 100644 --- a/ohsome/test/cassettes/test_response/test_elements_geometry.yaml +++ b/ohsome/test/cassettes/test_response/test_elements_geometry.yaml @@ -79,7 +79,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:05:08 GMT + - Thu, 16 Nov 2023 11:12:39 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_empty_geodataframe.yaml b/ohsome/test/cassettes/test_response/test_empty_geodataframe.yaml index a036deb..4241b51 100644 --- a/ohsome/test/cassettes/test_response/test_empty_geodataframe.yaml +++ b/ohsome/test/cassettes/test_response/test_empty_geodataframe.yaml @@ -41,7 +41,7 @@ interactions: Content-disposition: - attachment;filename=ohsome.geojson Date: - - Wed, 15 Nov 2023 12:05:11 GMT + - Thu, 16 Nov 2023 11:12:43 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_multi_index_false.yaml b/ohsome/test/cassettes/test_response/test_multi_index_false.yaml index 2940706..dfec525 100644 --- a/ohsome/test/cassettes/test_response/test_multi_index_false.yaml +++ b/ohsome/test/cassettes/test_response/test_multi_index_false.yaml @@ -128,7 +128,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:06 GMT + - Thu, 16 Nov 2023 11:12:36 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_not_implemented_query.yaml b/ohsome/test/cassettes/test_response/test_not_implemented_query.yaml index d1d961c..984aed7 100644 --- a/ohsome/test/cassettes/test_response/test_not_implemented_query.yaml +++ b/ohsome/test/cassettes/test_response/test_not_implemented_query.yaml @@ -50,7 +50,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:04 GMT + - Thu, 16 Nov 2023 11:12:34 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/cassettes/test_response/test_users_timestamp.yaml b/ohsome/test/cassettes/test_response/test_users_timestamp.yaml index 6275210..ae48d1c 100644 --- a/ohsome/test/cassettes/test_response/test_users_timestamp.yaml +++ b/ohsome/test/cassettes/test_response/test_users_timestamp.yaml @@ -43,7 +43,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 15 Nov 2023 12:05:10 GMT + - Thu, 16 Nov 2023 11:12:41 GMT Keep-Alive: - timeout=5, max=100 Server: diff --git a/ohsome/test/test_client.py b/ohsome/test/test_client.py index 4ad8bbd..05d98cf 100644 --- a/ohsome/test/test_client.py +++ b/ohsome/test/test_client.py @@ -9,7 +9,6 @@ import geopandas as gpd import pandas as pd import pytest -import numpy as np import ohsome from ohsome.constants import OHSOME_VERSION @@ -62,34 +61,6 @@ def test_check_time_parameter_list(base_client): client.elements.count.post(bcircles=bcircles, time=time, filter=fltr) -@pytest.mark.vcr -def test_check_time_parameter_datetimeindex(base_client): - """ - Checks whether time provided as pandas.DateTimeIndex is converted correctly - :return: - """ - time = pd.date_range("2018-01-01", periods=3, freq="D") - bcircles = gpd.read_file(f"{script_path}/data/points.geojson") - fltr = "amenity=restaurant and type:way" - - client = base_client - client.elements.count.post(bcircles=bcircles, time=time, filter=fltr) - - -@pytest.mark.vcr -def test_check_time_parameter_series(base_client): - """ - Checks whether time provided as pandas.DateTimeIndex is converted correctly - :return: - """ - time = pd.Series(["2018-01-01", "2018-01-02"]) - bcircles = gpd.read_file(f"{script_path}/data/points.geojson") - fltr = "amenity=restaurant and type:way" - - client = base_client - client.elements.count.post(bcircles=bcircles, time=time, filter=fltr) - - @pytest.mark.vcr def test_check_time_parameter_datetime(base_client): """ @@ -330,22 +301,6 @@ def test_format_bboxes_list(base_client): client.elements.count.post(bboxes=bboxes, time=time, filter=fltr) -@pytest.mark.vcr -def test_bbox_numpy(base_client): - """ - Tests whether numpy arrays are supported as input parameters - :return: - """ - - time = "2010-01-01" - fltr = "amenity=restaurant and type:node" - - client = base_client - - bboxes = np.array([8.67066, 49.41423, 8.68177, 49.4204]) - client.elements.count.post(bboxes=bboxes, time=time, filter=fltr) - - @pytest.mark.vcr def test_post_with_endpoint_string(base_client): """ diff --git a/ohsome/test/test_exceptions.py b/ohsome/test/test_exceptions.py index 6628193..d7689fd 100644 --- a/ohsome/test/test_exceptions.py +++ b/ohsome/test/test_exceptions.py @@ -75,7 +75,7 @@ def test_invalid_endpoint(): @pytest.mark.vcr def test_disable_logging(base_client): """ - Tests whether logging is disabled so no new log file if created if an OhsomeException occurs + Tests whether logging is disabled so no new log file is created if an OhsomeException occurs :return: """ base_client.log = False diff --git a/ohsome/test/test_helper.py b/ohsome/test/test_helper.py index 8208a62..6a00a3d 100644 --- a/ohsome/test/test_helper.py +++ b/ohsome/test/test_helper.py @@ -2,11 +2,21 @@ # -*- coding: utf-8 -*- """Tests for utility functions""" - -from ohsome.helper import find_groupby_names, extract_error_message_from_invalid_json -import os +import datetime import logging +import os +import numpy as np +import pandas as pd +import pytest + +from ohsome.helper import ( + find_groupby_names, + extract_error_message_from_invalid_json, + format_time, + convert_arrays, + format_list_parameters, +) script_path = os.path.dirname(os.path.realpath(__file__)) logger = logging.getLogger(__name__) @@ -45,7 +55,13 @@ def test_extract_error_message_from_invalid_json(): invalid_response_text = src.read() expected_error_code = 500 - expected_message = 'A broken response has been received: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.sql.SQLTransientConnectionException: HikariPool-1 - Connection is not available, request timed out after 30000ms.; "error" : "OK"; "timestamp" : "2020-05-19T07:07:25.356+0000"; "path" : "/elements/geometry"; "status" : 200' + expected_message = ( + "A broken response has been received: java.lang.RuntimeException: " + "java.lang.RuntimeException: java.lang.RuntimeException: " + "java.sql.SQLTransientConnectionException: HikariPool-1 - Connection is not available, request " + 'timed out after 30000ms.; "error" : "OK"; "timestamp" : "2020-05-19T07:07:25.356+0000"; ' + '"path" : "/elements/geometry"; "status" : 200' + ) error_code, message = extract_error_message_from_invalid_json(invalid_response_text) @@ -83,7 +99,10 @@ def test_extract_error_message_from_invalid_json_outOfMemory(): invalid_response_text = src.read() expected_error_code = 507 - expected_message = 'A broken response has been received: java.lang.OutOfMemoryError; "error" : "OK"; "timestamp" : "2021-06-01T11:38:52.821+0000"; "path" : "/elements/geometry"; "status" : 200' + expected_message = ( + 'A broken response has been received: java.lang.OutOfMemoryError; "error" : "OK"; ' + '"timestamp" : "2021-06-01T11:38:52.821+0000"; "path" : "/elements/geometry"; "status" : 200' + ) error_code, message = extract_error_message_from_invalid_json(invalid_response_text) @@ -102,9 +121,92 @@ def test_extract_error_message_from_invalid_json_custonErrorCode(): invalid_response_text = src.read() expected_error_code = 413 - expected_message = 'A broken response has been received: The given query is too large in respect to the given timeout. Please use a smaller region and/or coarser time period.; "timestamp" : "2021-06-02T10:07:46.438591"; "requestUrl" : "http://localhost:8080/elements/geometry"; "status" : 413' + expected_message = ( + "A broken response has been received: The given query is too large in respect to the given " + 'timeout. Please use a smaller region and/or coarser time period.; "timestamp" : ' + '"2021-06-02T10:07:46.438591"; "requestUrl" : "http://localhost:8080/elements/geometry"; ' + '"status" : 413' + ) error_code, message = extract_error_message_from_invalid_json(invalid_response_text) assert error_code == expected_error_code assert message == expected_message + + +def test_array_conversion(): + """Tests whether numpy arrays are supported as input parameters""" + method_input = {"bbox": np.ones(3)} + + output = convert_arrays(method_input) + + assert isinstance(output["bbox"], list) + + +def test_convert_arrays_multi_dim(): + """Test error raising on multi dim array.""" + method_input = {"bbox": np.ndarray(shape=(2, 2))} + with pytest.raises( + AssertionError, + match="Only one dimensional arrays are supported for parameter bbox", + ): + convert_arrays(method_input) + + +def test_format_time(): + """Test if the time formatter covers all cases.""" + parameters = { + "time_str": {"input": "2022-01-01", "output": "2022-01-01"}, + "time_datetime": { + "input": datetime.datetime(2022, 1, 1), + "output": "2022-01-01T00:00:00", + }, + "time_date": {"input": datetime.date(2022, 1, 1), "output": "2022-01-01"}, + "time_list": { + "input": ["2022-01-01", "2022-01-02"], + "output": "2022-01-01,2022-01-02", + }, + "time_pandas_index": { + "input": pd.date_range("2022-01-01", periods=2, freq="D"), + "output": "2022-01-01T00:00:00,2022-01-02T00:00:00", + }, + "time_pandas_series": { + "input": pd.Series(["2022-01-01", "2022-01-02"]), + "output": "2022-01-01,2022-01-02", + }, + } + + for k, v in parameters.items(): + output = format_time(v["input"]) + assert v["output"] == output, f"Input type {k} not correctly formatted." + + +def test_format_time_error_format_not_supported(): + """Test weather a time with wrong type (e.g. a dict) raises an error.""" + with pytest.raises( + ValueError, + match="The given time format is not supported. Feel free to open an " + "issue in the ohsome-py repository for a feature request.", + ): + format_time({}) + + +def test_format_list_parameters(): + """Test if formatting of list-like parameters works, and does not affect other parameters.""" + method_input = { + "groupByKeys": ["k1", "k2"], + "groupByValues": ["v1", "v2"], + "otherParam": ["l"], + "properties": ["p1"], + } + + expected_output = { + "groupByKeys": "k1,k2", + "groupByValues": "v1,v2", + "otherParam": ["l"], + "properties": "p1", + } + + output = format_list_parameters(method_input) + + assert output == expected_output