Fix memory issues for Spark (#1478)

coiled · Mar 20, 2024 · b85b497 · b85b497
1 parent 6be0edb
commit b85b497
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/ci/requirements-2tpch-non-dask.in b/ci/requirements-2tpch-non-dask.in
@@ -10,5 +10,8 @@ grpcio==1.62.1
 grpcio-status==1.62.1
 protobuf==4.25.2
 
+# First version to be able to set memory factors
+coiled>=1.13.1.dev1
+
 # Other TPCH tests
 polars==0.20.13
diff --git a/tests/tpch/conftest.py b/tests/tpch/conftest.py
@@ -250,8 +250,7 @@ def spark_setup(cluster, local):
         )
         spark_dashboard = parse_url("http://localhost:4040")
     else:
-        spark = cluster.get_spark()
-
+        spark = cluster.get_spark(executor_memory_factor=0.8, worker_memory_factor=0.9)
         # Available on coiled>=1.12.4
         if not hasattr(cluster, "_spark_dashboard"):
             cluster._spark_dashboard = (