Pagination of API requests

.gitignore

@@ -2,3 +2,4 @@ cloud-config-vultr.ini
 output
 resources
 vultr-cloud-*.tar.gz
+.vscode/

plugins/doc_fragments/vultr_v2.py

@@ -38,6 +38,15 @@ class ModuleDocFragment(object):
       - Fallback environment variable C(VULTR_API_RETRY_MAX_DELAY).
     type: int
     default: 12
+  api_results_per_page:
+    description:
+      - When receiving large numbers of resources, specify how many results should be returned per call to API.
+      - This does not determine how many results are returned; all resources are returned according to other filters.
+      - Vultr API maximum is 500.
+      - Fallback environment variable C(VULTR_API_RESULTS_PER_PAGE).
+    type: int
+    default: 100
+    version_added: 1.14.0
   api_endpoint:
     description:
       - URL to API endpint (without trailing slash).

plugins/module_utils/vultr_v2.py

@@ -11,6 +11,7 @@
 
 from ansible.module_utils._text import to_text
 from ansible.module_utils.basic import env_fallback
+from ansible.module_utils.common.dict_transformations import dict_merge
 from ansible.module_utils.six.moves.urllib.parse import quote
 from ansible.module_utils.urls import fetch_url
 
@@ -45,6 +46,11 @@ def vultr_argument_spec():
             fallback=(env_fallback, ["VULTR_API_RETRY_MAX_DELAY"]),
             default=12,
         ),
+        api_results_per_page=dict(
+            type="int",
+            fallback=(env_fallback, ["VULTR_API_RESULTS_PER_PAGE"]),
+            default=100,
+        ),
         validate_certs=dict(
             type="bool",
             default=True,
@@ -113,6 +119,7 @@ def __init__(
                 "api_timeout": module.params["api_timeout"],
                 "api_retries": module.params["api_retries"],
                 "api_retry_max_delay": module.params["api_retry_max_delay"],
+                "api_results_per_page": module.params["api_results_per_page"],
                 "api_endpoint": module.params["api_endpoint"],
             },
         }
@@ -135,6 +142,35 @@ def transform_resource(self, resource):
         """
         return resource
 
+    def paginate_api_query(self, path, method="GET", data=None, query_params=None, result_key=None):
+        result_key = result_key or self.ressource_result_key_plural
+        pager_param = dict(per_page=self.module.params["api_results_per_page"])
+        if query_params is not None:
+            query_params = dict_merge(query_params, pager_param)
+        else:
+            query_params = pager_param
+
+        result = dict()
+        cursor = dict()
+        while True:
+            resp = self.api_query(
+                path=path,
+                method=method,
+                data=data,
+                query_params=dict_merge(query_params, cursor),
+            )
+
+            resp_body = resp.get(result_key, {})
+            if isinstance(resp_body, list):
+                if isinstance(result.get(result_key), list):
+                    result[result_key].extend(resp_body)
+                else:
+                    result[result_key] = resp_body
+            cursor["cursor"] = resp.get("meta", {}).get("links", {}).get("next", "")
+
+            if cursor["cursor"] == "":
+                return result
+
     def api_query(self, path, method="GET", data=None, query_params=None):
         if query_params:
             query = "?"
@@ -151,12 +187,12 @@ def api_query(self, path, method="GET", data=None, query_params=None):
         resp_body = None
         for retry in range(0, self.module.params["api_retries"]):
             resp, info = fetch_url(
-                self.module,
-                self.module.params["api_endpoint"] + path,
+                module=self.module,
+                url=self.module.params["api_endpoint"] + path,
                 method=method,
                 data=data,
                 headers=self.headers,
-                timeout=self.module.params["api_timeout"],
+                timeout=int(self.module.params["api_timeout"]),
             )
 
             resp_body = resp.read() if resp is not None else ""
@@ -254,7 +290,10 @@ def query_by_id(self, resource_id=None, path=None, result_key=None, skip_transfo
         path = path or self.resource_path
         result_key = result_key or self.ressource_result_key_singular
 
-        resource = self.api_query(path="%s%s" % (path, "/" + resource_id if resource_id else resource_id))
+        resource = self.api_query(
+            path="%s%s" % (path, "/" + resource_id if resource_id else resource_id)
+        )
+
         if resource:
             if skip_transform:
                 return resource[result_key]
@@ -272,7 +311,11 @@ def query_list(self, path=None, result_key=None, query_params=None):
         path = path or self.resource_path
         result_key = result_key or self.ressource_result_key_plural
 
-        resources = self.api_query(path=path, query_params=query_params)
+        resources = self.paginate_api_query(
+            path=path,
+            query_params=query_params,
+            result_key=result_key,
+        )
         return resources[result_key] if resources else []
 
     def wait_for_state(self, resource, key, states, cmp="=", retries=60, skip_wait=False):

plugins/modules/account_info.py

@@ -58,6 +58,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/bare_metal.py

@@ -177,6 +177,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/block_storage.py

@@ -120,6 +120,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/block_storage_info.py

@@ -55,6 +55,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/dns_domain.py

@@ -82,6 +82,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/dns_domain_info.py

@@ -55,6 +55,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/dns_record.py

@@ -148,6 +148,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/firewall_group.py

@@ -67,6 +67,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/firewall_group_info.py

@@ -55,6 +55,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/firewall_rule.py

@@ -122,6 +122,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/firewall_rule_info.py

@@ -59,6 +59,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/instance.py

@@ -221,6 +221,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/instance_info.py

@@ -72,6 +72,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/object_storage.py

@@ -81,6 +81,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/object_storage_cluster_info.py

@@ -56,6 +56,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/object_storage_info.py

@@ -56,6 +56,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/os_info.py

@@ -56,6 +56,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/plan_info.py

@@ -56,6 +56,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/plan_metal_info.py

@@ -58,6 +58,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/region_info.py

@@ -56,6 +56,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/reserved_ip.py

@@ -101,6 +101,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success

plugins/modules/snapshot.py

@@ -85,6 +85,11 @@
       returned: success
       type: int
       sample: 12
+    api_results_per_page:
+      description: Number of results returned per call to API.
+      returned: success
+      type: int
+      sample: 100
     api_endpoint:
       description: Endpoint used for the API requests.
       returned: success