Add new CA certs support

Dockerfile

@@ -60,7 +60,7 @@ RUN pip install requests-toolbelt>=1.0.0
 
 RUN pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html
 
-RUN pip install supervisely==6.72.127
+RUN pip install supervisely==6.72.142
 # for development
 # RUN pip install git+https://github.com/supervisely/supervisely.git@minor-improvements
 

agent/worker/constants.py

@@ -31,6 +31,7 @@
 _APP_DEBUG_DOCKER_IMAGE = "APP_DEBUG_DOCKER_IMAGE"
 
 _REQUESTS_CA_BUNDLE = "REQUESTS_CA_BUNDLE"
+_REQUESTS_CA_BUNDLE_DIR_CONTAINER = "REQUESTS_CA_BUNDLE_DIR_CONTAINER"
 _HOST_REQUESTS_CA_BUNDLE = "HOST_REQUESTS_CA_BUNDLE"
 _SSL_CERT_FILE = "SSL_CERT_FILE"
 
@@ -109,6 +110,7 @@
     _DEFAULT_APP_DOCKER_IMAGE: "supervisely/base-py-sdk",
     _AGENT_FILES_IN_APP_CONTAINER: "/agent-storage",
     _AUTO_CLEAN_INT_RANGE_DAYS: 7,
+    _REQUESTS_CA_BUNDLE_DIR_CONTAINER: "/sly_certs",
 }
 
 
@@ -395,9 +397,40 @@ def APP_DEBUG_DOCKER_IMAGE():
 
 
 def REQUESTS_CA_BUNDLE():
+    """Certs file in Agent container"""
     return read_optional_setting(_REQUESTS_CA_BUNDLE)
 
 
+def REQUESTS_CA_BUNDLE_DIR():
+    """DIR where REQUESTS_CA_BUNDLE stored"""
+    if REQUESTS_CA_BUNDLE() is not None:
+        return os.path.dirname(REQUESTS_CA_BUNDLE())
+    return None
+
+
+def REQUESTS_CA_BUNDLE_DIR_CONTAINER():
+    """DIR where REQUESTS_CA_BUNDLE stored in App container"""
+    return read_optional_setting(_REQUESTS_CA_BUNDLE_DIR_CONTAINER)
+
+
+def REQUESTS_CA_BUNDLE_CONTAINER():
+    """Certs file in App container"""
+    if REQUESTS_CA_BUNDLE() is not None:
+        filename = sly.fs.get_file_name_with_ext(REQUESTS_CA_BUNDLE())
+        return os.path.join(REQUESTS_CA_BUNDLE_DIR_CONTAINER(), filename)
+    return None
+
+
+def MOUNTED_REQUESTS_CA_BUNDLE_DIR():
+    """Certs file path in mounted volume inside Agent container."""
+    return os.path.join(AGENT_ROOT_DIR(), "certs")
+
+
+def MOUNTED_HOST_REQUESTS_CA_BUNDLE():
+    """Certs file path in mounted volume on Host."""
+    return os.path.join(HOST_DIR(), "certs")
+
+
 def HOST_REQUESTS_CA_BUNDLE():
     return read_optional_setting(_HOST_REQUESTS_CA_BUNDLE)
 
@@ -498,3 +531,13 @@ def init_constants():
         sly.fs.mkdir(SUPERVISELY_AGENT_FILES_CONTAINER())
     if SUPERVISELY_SYNCED_APP_DATA_CONTAINER() is not None:
         sly.fs.mkdir(SUPERVISELY_SYNCED_APP_DATA_CONTAINER())
+
+    if REQUESTS_CA_BUNDLE() is not None:
+        # check if certs not in mounted folder
+        if REQUESTS_CA_BUNDLE_DIR() != MOUNTED_REQUESTS_CA_BUNDLE_DIR():
+            filename = sly.fs.get_file_name_with_ext(REQUESTS_CA_BUNDLE())
+            sly.fs.mkdir(MOUNTED_REQUESTS_CA_BUNDLE_DIR())
+            sly.fs.copy_file(
+                REQUESTS_CA_BUNDLE(),
+                os.path.join(MOUNTED_REQUESTS_CA_BUNDLE_DIR(), filename),
+            )

agent/worker/task_app.py

@@ -290,9 +290,9 @@ def _get_task_volumes(self):
                 "mode": "rw",
             }
 
-        if constants.HOST_REQUESTS_CA_BUNDLE() is not None:
-            res[constants.HOST_REQUESTS_CA_BUNDLE()] = {
-                "bind": constants.REQUESTS_CA_BUNDLE(),
+        if constants.REQUESTS_CA_BUNDLE() is not None:
+            res[constants.MOUNTED_HOST_REQUESTS_CA_BUNDLE()] = {
+                "bind": constants.REQUESTS_CA_BUNDLE_DIR_CONTAINER(),
                 "mode": "ro",
             }
 
@@ -524,7 +524,7 @@ def _exec_command(self, command, add_envs=None, container_id=None):
                 "SERVER_ADDRESS": self.info["server_address"],
                 "API_TOKEN": self.info["api_token"],
                 "AGENT_TOKEN": constants.TOKEN(),
-                constants._REQUESTS_CA_BUNDLE: constants.REQUESTS_CA_BUNDLE(),
+                constants._REQUESTS_CA_BUNDLE: constants.REQUESTS_CA_BUNDLE_CONTAINER(),
                 "PIP_ROOT_USER_ACTION": "ignore",
                 **add_envs,
             },

agent/worker/task_dockerized.py

@@ -123,6 +123,10 @@ def main_step_envs(self):
             env_val = os.getenv(env_key)
             if env_val is not None:
                 envs[env_key] = env_val
+
+        if constants.REQUESTS_CA_BUNDLE() is not None:
+            envs["REQUESTS_CA_BUNDLE"] = constants.REQUESTS_CA_BUNDLE_IN_CONTAINER()
+
         return envs
 
     def main_step(self):
@@ -148,11 +152,12 @@ def _get_task_volumes(self):
         volumes = {
             self.dir_task_host: {"bind": "/sly_task_data", "mode": "rw"},
         }
-        if constants.HOST_REQUESTS_CA_BUNDLE() is not None:
-            volumes[constants.HOST_REQUESTS_CA_BUNDLE()] = {
-                "bind": constants.REQUESTS_CA_BUNDLE(),
+        if constants.REQUESTS_CA_BUNDLE() is not None:
+            volumes[constants.MOUNTED_HOST_REQUESTS_CA_BUNDLE()] = {
+                "bind": constants.REQUESTS_CA_BUNDLE_DIR_CONTAINER(),
                 "mode": "ro",
             }
+
         return volumes
 
     def get_spawn_entrypoint(self):
@@ -188,7 +193,7 @@ def spawn_container(self, add_envs=None, add_labels=None, entrypoint_func=None):
                 constants._HTTP_PROXY.lower(): constants.HTTP_PROXY(),
                 constants._HTTPS_PROXY.lower(): constants.HTTPS_PROXY(),
                 constants._NO_PROXY.lower(): constants.NO_PROXY(),
-                constants._REQUESTS_CA_BUNDLE: constants.REQUESTS_CA_BUNDLE(),
+                constants._REQUESTS_CA_BUNDLE: constants.REQUESTS_CA_BUNDLE_CONTAINER(),
                 "PIP_ROOT_USER_ACTION": "ignore",
                 **add_envs,
             }

requirements.txt

@@ -7,7 +7,7 @@ grpcio-tools==1.47.0
 # py3exiv2==0.9.3
 packaging==21.3
 version-parser==1.0.1
-supervisely==6.72.127
+supervisely==6.72.142
 docker==3.3.0
 black
 python-slugify==6.1.2