Login when docker pull failed with no basic auth credentials error (#87)

* login when docker pull failed with no basic auth credentials error * imports bugfix --------- Co-authored-by: Tony Bartsits <[email protected]>
supervisely · Aug 30, 2024 · ff3eed1 · ff3eed1
1 parent 8012249
commit ff3eed1
Show file tree

Hide file tree

Showing 5 changed files with 92 additions and 36 deletions.
diff --git a/agent/worker/agent.py b/agent/worker/agent.py
@@ -17,6 +17,7 @@
 from pathlib import Path
 from filelock import FileLock
 from datetime import datetime
+from docker.errors import DockerException
 
 import supervisely_lib as sly
 
@@ -450,22 +451,7 @@ def _stop_missed_containers(self, ecosystem_token):
             )
 
     def _docker_login(self):
-        doc_logs = constants.DOCKER_LOGIN().split(",")
-        doc_pasws = constants.DOCKER_PASSWORD().split(",")
-        doc_regs = constants.DOCKER_REGISTRY().split(",")
-
-        for login, password, registry in zip(doc_logs, doc_pasws, doc_regs):
-            if registry:
-                try:
-                    doc_login = self.docker_api.login(
-                        username=login, password=password, registry=registry
-                    )
-                    self.logger.info(
-                        "DOCKER_CLIENT_LOGIN_SUCCESS", extra={**doc_login, "registry": registry}
-                    )
-                except Exception as e:
-                    if not constants.OFFLINE_MODE():
-                        raise e
+        agent_utils.docker_login(self.docker_api, self.logger)
 
     def submit_log(self):
         while True:
@@ -652,13 +638,31 @@ def update_base_layers(self):
                     )
                     image = f"{constants.SLY_APPS_DOCKER_REGISTRY()}/{image}"
 
-                docker_utils.docker_pull_if_needed(
-                    self.docker_api,
-                    image,
-                    policy=docker_utils.PullPolicy.ALWAYS,
-                    logger=self.logger,
-                    progress=False,
-                )
+                try:
+                    docker_utils.docker_pull_if_needed(
+                        self.docker_api,
+                        image,
+                        policy=docker_utils.PullPolicy.ALWAYS,
+                        logger=self.logger,
+                        progress=False,
+                    )
+                except DockerException as e:
+                    if "no basic auth credentials" in str(e).lower():
+                        self.logger.warn(
+                            f"Failed to pull docker image '{image}'. Will try to login and pull again",
+                            exc_info=True,
+                        )
+                        self._docker_login()
+                        docker_utils.docker_pull_if_needed(
+                            self.docker_api,
+                            image,
+                            policy=docker_utils.PullPolicy.ALWAYS,
+                            logger=self.logger,
+                            progress=False,
+                        )
+                    else:
+                        raise e
+
                 self.logger.info(f"Docker image '{image}' has been pulled successfully")
                 pulled.append(image)
             except:

diff --git a/agent/worker/agent_utils.py b/agent/worker/agent_utils.py
@@ -1070,3 +1070,22 @@ def maybe_update_runtime():
 def convert_millicores_to_cpu_quota(millicores, cpu_period=100000):
     cpu_quota = (millicores / 1000) * cpu_period
     return int(cpu_quota)
+
+
+def docker_login(docker_api, logger):
+    doc_logs = constants.DOCKER_LOGIN().split(",")
+    doc_pasws = constants.DOCKER_PASSWORD().split(",")
+    doc_regs = constants.DOCKER_REGISTRY().split(",")
+
+    for login, password, registry in zip(doc_logs, doc_pasws, doc_regs):
+        if registry:
+            try:
+                doc_login = docker_api.login(
+                    username=login, password=password, registry=registry
+                )
+                logger.info(
+                    "DOCKER_CLIENT_LOGIN_SUCCESS", extra={**doc_login, "registry": registry}
+                )
+            except Exception as e:
+                if not constants.OFFLINE_MODE():
+                    raise e
diff --git a/agent/worker/docker_utils.py b/agent/worker/docker_utils.py
@@ -98,10 +98,11 @@ def _docker_pull(docker_api, docker_image_name, logger, raise_exception=True):
         )
     except DockerException as e:
         if raise_exception is True:
-            raise DockerException(
-                "Unable to pull image: see actual error above. "
-                "Please, run the task again or contact support team."
-            )
+            raise e
+            # raise DockerException(
+            #     "Unable to pull image: see actual error above. "
+            #     "Please, run the task again or contact support team."
+            # )
         else:
             logger.warn("Pulling step is skipped. Unable to pull image: {!r}.".format(str(e)))
 

diff --git a/agent/worker/task_app.py b/agent/worker/task_app.py
@@ -15,6 +15,7 @@
 from slugify import slugify
 from pathlib import Path
 from packaging import version
+from worker import agent_utils
 from version_parser import Version
 from enum import Enum
 from typing import Optional
@@ -521,12 +522,28 @@ def sync_pip_cache(self):
     @handle_exceptions
     def find_or_run_container(self):
         add_labels = {"sly_app": "1", "app_session_id": str(self.info["task_id"])}
-        docker_utils.docker_pull_if_needed(
-            self._docker_api,
-            self.docker_image_name,
-            constants.PULL_POLICY(),
-            self.logger,
-        )
+        try:
+            docker_utils.docker_pull_if_needed(
+                self._docker_api,
+                self.docker_image_name,
+                constants.PULL_POLICY(),
+                self.logger,
+            )
+        except DockerException as e:
+            if "no basic auth credentials" in str(e).lower():
+                self.logger.warn(
+                    f"Failed to pull docker image '{self.docker_image_name}'. Will try to login and pull again",
+                    exc_info=True,
+                )
+                agent_utils.docker_login(self.docker_api, self.logger)
+                docker_utils.docker_pull_if_needed(
+                    self._docker_api,
+                    self.docker_image_name,
+                    constants.PULL_POLICY(),
+                    self.logger,
+                )
+            else:
+                raise e
         self.sync_pip_cache()
         if self._container is None:
             try:

diff --git a/agent/worker/task_pull_docker_image.py b/agent/worker/task_pull_docker_image.py
@@ -1,7 +1,9 @@
 # coding: utf-8
+from docker.errors import DockerException
 from packaging import version
 import supervisely_lib as sly
 
+from worker import agent_utils
 from worker import constants
 from worker import docker_utils
 from worker.task_sly import TaskSly
@@ -29,9 +31,22 @@ def docker_api(self, val):
 
     def task_main_func(self):
         self.logger.info("TASK_START", extra={"event_type": sly.EventType.TASK_STARTED})
-        docker_utils.docker_pull_if_needed(
-            self._docker_api, self.docker_image_name, self.info["pull_policy"], self.logger
-        )
+        try:
+            docker_utils.docker_pull_if_needed(
+                self._docker_api, self.docker_image_name, self.info["pull_policy"], self.logger
+            )
+        except DockerException as e:
+            if "no basic auth credentials" in str(e).lower():
+                self.logger.warn(
+                    f"Failed to pull docker image '{self.docker_image_name}'. Will try to login and pull again",
+                    exc_info=True,
+                )
+                agent_utils.docker_login(self.docker_api, self.logger)
+                docker_utils.docker_pull_if_needed(
+                    self._docker_api, self.docker_image_name, self.info["pull_policy"], self.logger
+                )
+            else:
+                raise e
         docker_img = self._docker_api.images.get(self.docker_image_name)
         if constants.CHECK_VERSION_COMPATIBILITY():
             self._validate_version(