Config: Dune Request Timeout (#122)

This PR closes #121 and adds an option to set a custom request timeout when uploading data to Dune.

---------

Co-authored-by: Felix Leupold <[email protected]>
Co-authored-by: Benjamin Smith <[email protected]>

README.md

@@ -66,6 +66,7 @@ For Postgres sources (`ref: Postgres`):
 
 For Dune destinations (`ref: Dune`):
 - `table_name`: String. Name of Dune table to update
+- `request_timeout`: [optional] Request timeout for the dune client
 
 For Postgres destinations (`ref: Postgres`):
 - `table_name`: String. Name of table to insert/append into

src/config.py

@@ -322,9 +322,22 @@ def _build_destination(
 
         match dest.type:
             case Database.DUNE:
+                try:
+                    request_timeout = dest_config["timeout"]
+                    request_timeout = int(request_timeout)
+                except KeyError:
+                    log.debug("Dune request timeout not set: defaulting to 10")
+                    request_timeout = 10
+                except ValueError as e:
+                    log.error(
+                        "request_timeout parameter must be a number, received type %s",
+                        type(request_timeout),
+                    )
+                    raise e
                 return DuneDestination(
                     api_key=dest.key,
                     table_name=dest_config["table_name"],
+                    request_timeout=request_timeout,
                 )
 
             case Database.POSTGRES:

src/destinations/dune.py

@@ -20,10 +20,15 @@ class DuneDestination(Destination[DataFrame]):
     table_name : str
         The name of the table where the query results will be stored.
 
+    [optional]
+    request_timeout : int
+        Default: 10
+        The request timeout for the dune client.
+
     """
 
-    def __init__(self, api_key: str, table_name: str):
-        self.client = DuneClient(api_key)
+    def __init__(self, api_key: str, table_name: str, request_timeout: int):
+        self.client = DuneClient(api_key, request_timeout=request_timeout)
         self.table_name: str = table_name
         super().__init__()
 

tests/fixtures/config/postgres_to_dune.yaml

@@ -17,3 +17,4 @@ jobs:
     destination:
       ref: dune
       table_name: table_name
+      request_timeout: 600

tests/unit/destinations_test.py

@@ -40,6 +40,21 @@ def test_ensure_index_disabled_when_uploading(self, mock_to_csv, *_):
         destination.save(dummy_df)
         mock_to_csv.assert_called_once_with(index=False)
 
+    @patch("pandas.core.generic.NDFrame.to_csv", name="Fake csv writer")
+    def test_duneclient_sets_timeout(self, mock_to_csv, *_):
+        dummy_data = [
+            {"foo": "bar"},
+            {"baz": "daz"},
+        ]
+        dummy_df = pd.DataFrame(dummy_data)
+        for timeout in [1, 10, 100, 1000, 1500]:
+            destination = DuneDestination(
+                api_key=os.getenv("DUNE_API_KEY"),
+                table_name="foo",
+                request_timeout=timeout,
+            )
+            assert destination.client.request_timeout == timeout
+
     @patch("dune_client.api.table.TableAPI.upload_csv", name="Fake CSV uploader")
     def test_dune_error_handling(self, mock_dune_upload_csv):
         dest = DuneDestination(api_key="f00b4r", table_name="foo")