From b9c31e3918e7409c4cf2157a109ac7aeb9dffab1 Mon Sep 17 00:00:00 2001 From: Steven <steven.pi@studiosouza.be> Date: Fri, 15 Nov 2024 16:37:47 +0100 Subject: [PATCH] Added pagination to fetch_all --- .gitignore | 3 +++ gazu/client.py | 49 ++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 72364f99..3db1597b 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,6 @@ ENV/ # Rope project settings .ropeproject + +# PyCharm settings +.idea diff --git a/gazu/client.py b/gazu/client.py index 03de8d57..3e1c19ad 100644 --- a/gazu/client.py +++ b/gazu/client.py @@ -192,7 +192,7 @@ def get_full_url(path, client=default_client): def build_path_with_params(path, params): """ - Add params to a path using urllib encoding + Add params to a path using urllib encoding. Args: path (str): The url base path @@ -204,10 +204,18 @@ def build_path_with_params(path, params): if not params: return path - if hasattr(urllib, "urlencode"): - path = "%s?%s" % (path, urllib.urlencode(params)) - else: - path = "%s?%s" % (path, urllib.parse.urlencode(params)) + urlencode = ( + urllib.urlencode + if hasattr(urllib, "urlencode") + else urllib.parse.urlencode + ) + query_string = urlencode(params) + + if query_string: + # Support base paths that already contain query parameters. + path += "&" if "?" in path else "?" + path += query_string + return path @@ -400,16 +408,43 @@ def check_status(request, path, client=None): return status_code, False -def fetch_all(path, params=None, client=default_client): +def fetch_all(path, params=None, client=default_client, paginated=False): """ Args: path (str): The path for which we want to retrieve all entries. + paginated (bool): Will query the database in subsequently in batches of 100 entries at a time. Returns: list: All entries stored in database for a given model. You can add a filter to the model name like this: "tasks?project_id=project-id" """ - return get(url_path_join("data", path), params=params, client=client) + + if not params: + params = {} + if paginated: + params["page"] = 1 + + response = get(url_path_join("data", path), params=params, client=client) + + # Return non-paginated responses. + if not isinstance(response, dict) or not response.get("data", None): + return response + + total_pages = response.get("nb_pages", 1) + current_page = response.get("page", 1) + results = response.get("data", []) + + if current_page != total_pages: + for page in range(2, total_pages + 1): + response = get( + url_path_join("data", path), + params={"page": page}, + client=client, + ) + offset = response.get("offset", len(results)) + results = results[:offset] + [r for r in response.get("data", [])] + + return results def fetch_first(path, params=None, client=default_client):