Set default allow_large_pex to False

criteo · Sep 13, 2024 · e8d86d6 · e8d86d6
1 parent 0befd3a
commit e8d86d6
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/cluster_pack/skein/skein_config_builder.py b/cluster_pack/skein/skein_config_builder.py
@@ -67,7 +67,7 @@ def build(
         additional_files: Optional[List[str]] = None,
         tmp_dir: str = packaging._get_tmp_dir(),
         process_logs: Callable[[str], Any] = None,
-        allow_large_pex: bool = True
+        allow_large_pex: bool = False
 ) -> SkeinConfig:
     """Build the skein config for a module to execute
 
@@ -79,6 +79,9 @@ def build(
     :param tmp_dir: a temp dir for local files
     :param process_logs: hook with the local log path as a parameter,
                          can be used to uplaod the logs somewhere
+    :param allow_large_pex: Creates a non-executable pex that will need to be unzipped to circumvent
+                            python's limitation with zips > 2Gb. The file will need to be unzipped
+                            and the entry point will be <output>/__main__.py
     :return: SkeinConfig
     """
     if not package_path:

diff --git a/cluster_pack/skein/skein_launcher.py b/cluster_pack/skein/skein_launcher.py
@@ -26,7 +26,7 @@ def submit(skein_client: skein.Client,
            pre_script_hook: Optional[str] = None,
            max_attempts: int = 1, max_restarts: int = 0,
            process_logs: Callable[[str], Any] = None,
-           allow_large_pex: bool = True) -> str:
+           allow_large_pex: bool = False) -> str:
     """Execute a python module in a skein container
 
     :param skein_client: skein.Client to use
@@ -51,6 +51,9 @@ def submit(skein_client: skein.Client,
     :param max_restarts: maximum number of restarts allowed for the service
     :param process_logs: hook with the local log path as a parameter,
                          can be used to uplaod the logs somewhere
+    :param allow_large_pex: Creates a non-executable pex that will need to be unzipped to circumvent
+                            python's limitation with zips > 2Gb. The file will need to be unzipped
+                            and the entry point will be <output>/__main__.py
     :return: SkeinConfig
     """
     with tempfile.TemporaryDirectory() as tmp_dir:
@@ -61,7 +64,7 @@ def submit(skein_client: skein.Client,
             additional_files=additional_files,
             tmp_dir=tmp_dir,
             process_logs=process_logs,
-            allow_large_pex=True)
+            allow_large_pex=allow_large_pex)
 
         return _submit(
             skein_client, skein_config,