Add allow_large_pex to submit_func

cluster_pack/skein/skein_config_builder.py

@@ -20,7 +20,8 @@ def build_with_func(
         additional_files: Optional[List[str]] = None,
         tmp_dir: str = packaging._get_tmp_dir(),
         log_level: str = "INFO",
-        process_logs: Callable[[str], Any] = None
+        process_logs: Callable[[str], Any] = None,
+        allow_large_pex: bool = False
 ) -> SkeinConfig:
     """Build the skein config from provided a function
 
@@ -35,6 +36,9 @@ def build_with_func(
     :param log_level: default remote log level
     :param process_logs: hook with the local log path as a parameter,
                          can be used to uplaod the logs somewhere
+    :param allow_large_pex: Creates a non-executable pex that will need to be unzipped to circumvent
+                            python's limitation with zips > 2Gb. The file will need to be unzipped
+                            and the entry point will be <output>/__main__.py
     :return: SkeinConfig
     """
     function_name = f"function_{uuid.uuid4()}.dat"
@@ -57,7 +61,7 @@ def build_with_func(
         package_path,
         additional_files,
         tmp_dir,
-        process_logs)
+        process_logs, allow_large_pex=allow_large_pex)
 
 
 def build(

cluster_pack/skein/skein_launcher.py

@@ -86,7 +86,8 @@ def submit_func(skein_client: skein.Client,
                 acquire_map_reduce_delegation_token: bool = False,
                 pre_script_hook: Optional[str] = None,
                 max_attempts: int = 1, max_restarts: int = 0,
-                process_logs: Callable[[str], Any] = None) -> str:
+                process_logs: Callable[[str], Any] = None,
+                allow_large_pex: bool = False) -> str:
     """Submit a function in a skein container
 
     :param skein_client: skein.Client to use
@@ -111,6 +112,9 @@ def submit_func(skein_client: skein.Client,
     :param max_restarts: maximum number of restarts allowed for the service
     :param process_logs: hook with the local log path as a parameter,
                          can be used to uplaod the logs somewhere
+    :param allow_large_pex: Creates a non-executable pex that will need to be unzipped to circumvent
+                            python's limitation with zips > 2Gb. The file will need to be unzipped
+                            and the entry point will be <output>/__main__.py
     :return: SkeinConfig
     """
 
@@ -121,7 +125,8 @@ def submit_func(skein_client: skein.Client,
             package_path=package_path,
             additional_files=additional_files,
             tmp_dir=tmp_dir,
-            process_logs=process_logs)
+            process_logs=process_logs,
+            allow_large_pex=allow_large_pex)
 
         return _submit(
             skein_client, skein_config,