From ff3eed189d2896be6449da9e0423a2f1bf51ea87 Mon Sep 17 00:00:00 2001 From: Nikolai Petukhov Date: Fri, 30 Aug 2024 11:38:54 -0300 Subject: [PATCH] Login when docker pull failed with no basic auth credentials error (#87) * login when docker pull failed with no basic auth credentials error * imports bugfix --------- Co-authored-by: Tony Bartsits --- agent/worker/agent.py | 50 ++++++++++++++------------ agent/worker/agent_utils.py | 19 ++++++++++ agent/worker/docker_utils.py | 9 ++--- agent/worker/task_app.py | 29 +++++++++++---- agent/worker/task_pull_docker_image.py | 21 +++++++++-- 5 files changed, 92 insertions(+), 36 deletions(-) diff --git a/agent/worker/agent.py b/agent/worker/agent.py index eb1579c..16453cd 100644 --- a/agent/worker/agent.py +++ b/agent/worker/agent.py @@ -17,6 +17,7 @@ from pathlib import Path from filelock import FileLock from datetime import datetime +from docker.errors import DockerException import supervisely_lib as sly @@ -450,22 +451,7 @@ def _stop_missed_containers(self, ecosystem_token): ) def _docker_login(self): - doc_logs = constants.DOCKER_LOGIN().split(",") - doc_pasws = constants.DOCKER_PASSWORD().split(",") - doc_regs = constants.DOCKER_REGISTRY().split(",") - - for login, password, registry in zip(doc_logs, doc_pasws, doc_regs): - if registry: - try: - doc_login = self.docker_api.login( - username=login, password=password, registry=registry - ) - self.logger.info( - "DOCKER_CLIENT_LOGIN_SUCCESS", extra={**doc_login, "registry": registry} - ) - except Exception as e: - if not constants.OFFLINE_MODE(): - raise e + agent_utils.docker_login(self.docker_api, self.logger) def submit_log(self): while True: @@ -652,13 +638,31 @@ def update_base_layers(self): ) image = f"{constants.SLY_APPS_DOCKER_REGISTRY()}/{image}" - docker_utils.docker_pull_if_needed( - self.docker_api, - image, - policy=docker_utils.PullPolicy.ALWAYS, - logger=self.logger, - progress=False, - ) + try: + docker_utils.docker_pull_if_needed( + self.docker_api, + image, + policy=docker_utils.PullPolicy.ALWAYS, + logger=self.logger, + progress=False, + ) + except DockerException as e: + if "no basic auth credentials" in str(e).lower(): + self.logger.warn( + f"Failed to pull docker image '{image}'. Will try to login and pull again", + exc_info=True, + ) + self._docker_login() + docker_utils.docker_pull_if_needed( + self.docker_api, + image, + policy=docker_utils.PullPolicy.ALWAYS, + logger=self.logger, + progress=False, + ) + else: + raise e + self.logger.info(f"Docker image '{image}' has been pulled successfully") pulled.append(image) except: diff --git a/agent/worker/agent_utils.py b/agent/worker/agent_utils.py index 1c97162..95c9756 100644 --- a/agent/worker/agent_utils.py +++ b/agent/worker/agent_utils.py @@ -1070,3 +1070,22 @@ def maybe_update_runtime(): def convert_millicores_to_cpu_quota(millicores, cpu_period=100000): cpu_quota = (millicores / 1000) * cpu_period return int(cpu_quota) + + +def docker_login(docker_api, logger): + doc_logs = constants.DOCKER_LOGIN().split(",") + doc_pasws = constants.DOCKER_PASSWORD().split(",") + doc_regs = constants.DOCKER_REGISTRY().split(",") + + for login, password, registry in zip(doc_logs, doc_pasws, doc_regs): + if registry: + try: + doc_login = docker_api.login( + username=login, password=password, registry=registry + ) + logger.info( + "DOCKER_CLIENT_LOGIN_SUCCESS", extra={**doc_login, "registry": registry} + ) + except Exception as e: + if not constants.OFFLINE_MODE(): + raise e diff --git a/agent/worker/docker_utils.py b/agent/worker/docker_utils.py index d71582a..a3c34eb 100644 --- a/agent/worker/docker_utils.py +++ b/agent/worker/docker_utils.py @@ -98,10 +98,11 @@ def _docker_pull(docker_api, docker_image_name, logger, raise_exception=True): ) except DockerException as e: if raise_exception is True: - raise DockerException( - "Unable to pull image: see actual error above. " - "Please, run the task again or contact support team." - ) + raise e + # raise DockerException( + # "Unable to pull image: see actual error above. " + # "Please, run the task again or contact support team." + # ) else: logger.warn("Pulling step is skipped. Unable to pull image: {!r}.".format(str(e))) diff --git a/agent/worker/task_app.py b/agent/worker/task_app.py index 0191343..8144ed3 100644 --- a/agent/worker/task_app.py +++ b/agent/worker/task_app.py @@ -15,6 +15,7 @@ from slugify import slugify from pathlib import Path from packaging import version +from worker import agent_utils from version_parser import Version from enum import Enum from typing import Optional @@ -521,12 +522,28 @@ def sync_pip_cache(self): @handle_exceptions def find_or_run_container(self): add_labels = {"sly_app": "1", "app_session_id": str(self.info["task_id"])} - docker_utils.docker_pull_if_needed( - self._docker_api, - self.docker_image_name, - constants.PULL_POLICY(), - self.logger, - ) + try: + docker_utils.docker_pull_if_needed( + self._docker_api, + self.docker_image_name, + constants.PULL_POLICY(), + self.logger, + ) + except DockerException as e: + if "no basic auth credentials" in str(e).lower(): + self.logger.warn( + f"Failed to pull docker image '{self.docker_image_name}'. Will try to login and pull again", + exc_info=True, + ) + agent_utils.docker_login(self.docker_api, self.logger) + docker_utils.docker_pull_if_needed( + self._docker_api, + self.docker_image_name, + constants.PULL_POLICY(), + self.logger, + ) + else: + raise e self.sync_pip_cache() if self._container is None: try: diff --git a/agent/worker/task_pull_docker_image.py b/agent/worker/task_pull_docker_image.py index 384ff82..9193171 100644 --- a/agent/worker/task_pull_docker_image.py +++ b/agent/worker/task_pull_docker_image.py @@ -1,7 +1,9 @@ # coding: utf-8 +from docker.errors import DockerException from packaging import version import supervisely_lib as sly +from worker import agent_utils from worker import constants from worker import docker_utils from worker.task_sly import TaskSly @@ -29,9 +31,22 @@ def docker_api(self, val): def task_main_func(self): self.logger.info("TASK_START", extra={"event_type": sly.EventType.TASK_STARTED}) - docker_utils.docker_pull_if_needed( - self._docker_api, self.docker_image_name, self.info["pull_policy"], self.logger - ) + try: + docker_utils.docker_pull_if_needed( + self._docker_api, self.docker_image_name, self.info["pull_policy"], self.logger + ) + except DockerException as e: + if "no basic auth credentials" in str(e).lower(): + self.logger.warn( + f"Failed to pull docker image '{self.docker_image_name}'. Will try to login and pull again", + exc_info=True, + ) + agent_utils.docker_login(self.docker_api, self.logger) + docker_utils.docker_pull_if_needed( + self._docker_api, self.docker_image_name, self.info["pull_policy"], self.logger + ) + else: + raise e docker_img = self._docker_api.images.get(self.docker_image_name) if constants.CHECK_VERSION_COMPATIBILITY(): self._validate_version(