Merge pull request #148 from QuantConnect/bug-146-docker-pull-custom-…

…image

Do not pull local images

lean/commands/build.py

@@ -20,7 +20,7 @@
 from lean.click import LeanCommand, PathParameter
 from lean.container import container
 from lean.models.docker import DockerImage
-
+from lean.constants import CUSTOM_FOUNDATION, CUSTOM_RESEARCH, CUSTOM_ENGINE
 
 def _normalize_newlines(text: str) -> str:
     """Normalizes the newlines in a string to use \n (instead of \r or \r\n).
@@ -160,16 +160,16 @@ def build(root: Path, tag: str) -> None:
         foundation_image = DockerImage(name="quantconnect/lean", tag="foundation")
         container.docker_manager().pull_image(foundation_image)
     else:
-        foundation_image = DockerImage(name="lean-cli/foundation", tag=tag)
+        foundation_image = DockerImage(name=CUSTOM_FOUNDATION, tag=tag)
         _build_image(root, foundation_dockerfile, None, foundation_image)
 
     _compile_csharp(root, lean_dir, foundation_image)
     _compile_csharp(root, alpha_streams_dir, foundation_image)
 
-    custom_engine_image = DockerImage(name="lean-cli/engine", tag=tag)
+    custom_engine_image = DockerImage(name=CUSTOM_ENGINE, tag=tag)
     _build_image(root, lean_dir / "Dockerfile", foundation_image, custom_engine_image)
 
-    custom_research_image = DockerImage(name="lean-cli/research", tag=tag)
+    custom_research_image = DockerImage(name=CUSTOM_RESEARCH, tag=tag)
     _build_image(root, lean_dir / "DockerfileJupyter", custom_engine_image, custom_research_image)
 
     logger = container.logger()

lean/components/docker/docker_manager.py

@@ -35,7 +35,8 @@
 from lean.components.util.platform_manager import PlatformManager
 from lean.components.util.temp_manager import TempManager
 from lean.constants import SITE_PACKAGES_VOLUME_LIMIT, \
-    DOCKER_NETWORK
+    DOCKER_NETWORK, CUSTOM_FOUNDATION, CUSTOM_RESEARCH, CUSTOM_ENGINE
+
 from lean.models.docker import DockerImage
 from lean.models.errors import MoreInfoError
 from lean.components.util.custom_json_encoder import DecimalEncoder
@@ -59,8 +60,11 @@ def pull_image(self, image: DockerImage) -> None:
 
         :param image: the image to pull
         """
-        self._logger.info(f"Pulling {image}...")
+        if image.name == CUSTOM_RESEARCH or image.name == CUSTOM_ENGINE or image.name == CUSTOM_FOUNDATION:
+            self._logger.info(f"Skip pulling local image {image}...")
+            return
 
+        self._logger.info(f"Pulling {image}...")
         # We cannot really use docker_client.images.pull() here as it doesn't let us log the progress
         # Downloading multiple gigabytes without showing progress does not provide good developer experience
         # Since the pull command is the same on Windows, macOS and Linux we can safely use a system call
@@ -425,7 +429,7 @@ def is_missing_permission(self) -> bool:
             return "Permission denied" in str(exception)
         return False
 
-    def write_to_file(self, docker_container_name: str, docker_file: Path, data: Dict[str, Any]) -> None:        
+    def write_to_file(self, docker_container_name: str, docker_file: Path, data: Dict[str, Any]) -> None:
         """Write data to the file in docker.
 
         Args:
@@ -438,7 +442,7 @@ def write_to_file(self, docker_container_name: str, docker_file: Path, data: Dic
             raise ValueError(f"Container {docker_container_name} does not exist")
         if docker_container.status != "running":
             raise ValueError(f"Container {docker_container_name} is not running")
-            
+
         data = json.dumps(data, cls=DecimalEncoder)
         data = data.replace('"','\\"')
         command = f'docker exec {docker_container_name} bash -c "echo \'{data}\' > {docker_file.as_posix()}"'
@@ -448,7 +452,7 @@ def write_to_file(self, docker_container_name: str, docker_file: Path, data: Dic
             raise ValueError(f"Failed to write to {docker_file.name}: {exception.output.decode('utf-8')}")
         except Exception as e:
             raise ValueError(f"Failed to write to {docker_file.name}: {e}")
-    
+
     def read_from_file(self, docker_container_name: str, docker_file: Path, interval=1, timeout=30) -> Dict[str,Any]:
         """Read data from file in docker.
 
@@ -489,7 +493,7 @@ def read_from_file(self, docker_container_name: str, docker_file: Path, interval
                 error_message = "Rejected by Lean. Possible arguments error. Please check your logs and try again."
         if not success and not error_message:
             error_message = f"Failed to read result from docker file {docker_file.name} within {timeout} seconds"
-        
+
         return {
             "error": error_message,
             "success": success,

lean/constants.py

@@ -16,6 +16,11 @@
 
 # Due to the way the filesystem is mocked in unit tests, values should not be Path instances.
 
+# Custom images names
+CUSTOM_FOUNDATION = "lean-cli/foundation"
+CUSTOM_ENGINE = "lean-cli/engine"
+CUSTOM_RESEARCH = "lean-cli/research"
+
 # The python version of docker image
 LEAN_PYTHON_VERSION = "3.8"
 

tests/commands/test_build.py

@@ -21,10 +21,11 @@
 from lean.commands import lean
 from lean.container import container
 from lean.models.docker import DockerImage
+from lean.constants import CUSTOM_FOUNDATION, CUSTOM_RESEARCH, CUSTOM_ENGINE
 
-CUSTOM_FOUNDATION_IMAGE = DockerImage(name="lean-cli/foundation", tag="latest")
-CUSTOM_ENGINE_IMAGE = DockerImage(name="lean-cli/engine", tag="latest")
-CUSTOM_RESEARCH_IMAGE = DockerImage(name="lean-cli/research", tag="latest")
+CUSTOM_FOUNDATION_IMAGE = DockerImage(name=CUSTOM_FOUNDATION, tag="latest")
+CUSTOM_ENGINE_IMAGE = DockerImage(name=CUSTOM_ENGINE, tag="latest")
+CUSTOM_RESEARCH_IMAGE = DockerImage(name=CUSTOM_RESEARCH, tag="latest")
 
 
 def create_fake_repositories() -> None: