Clean Apps cache (#72)

* remove apps cache on apps cleaning

* remove clean cache from auto cleaning

* add debug log message

* freed space calculation bugfix

* add apps cache dir env var to the app task container

* add pylint ignore import error

agent/worker/agent_utils.py

@@ -12,7 +12,9 @@
 import io
 
 import supervisely_lib as sly
-from supervisely_lib._utils import get_certificates_list
+from supervisely_lib._utils import (
+    get_certificates_list,
+)  # pylint: disable=import-error, no-name-in-module
 
 from logging import Logger
 from typing import Callable, List, Optional, Tuple, Union, Container
@@ -239,20 +241,35 @@ def auto_clean(self, working_apps: Container[int]):
 
     def clean_all_app_data(self, working_apps: Optional[Container[int]] = None):
         self.logger.info("Cleaning apps data.")
-        self._apps_cleaner(working_apps, auto=False, clean_pip=False)
+        self._apps_cleaner(working_apps, auto=False, clean_pip=False, clean_apps_cache=True)
         self.clean_git_tags()
 
+    def clean_apps_cache(self):
+        cache_dir = constants.AGENT_APPS_CACHE_DIR()
+        cleaned_space = 0
+        for p in Path(cache_dir).iterdir():
+            if p.is_dir():
+                cleaned_space += sly.fs.get_directory_size(p.as_posix())
+                sly.fs.remove_dir(p)
+            else:
+                cleaned_space += sly.fs.get_file_size(p.as_posix())
+                p.unlink()
+        self.logger.debug("Apps cache cleaned. Space freed: %s bytes", cleaned_space)
+
     def _apps_cleaner(
         self,
         working_apps: Optional[Container[int]],
         auto: bool = False,
         clean_pip: bool = True,
+        clean_apps_cache: bool = False,
     ):
         cleaned_sessions = self.clean_app_sessions(auto=auto, working_apps=working_apps)
         if auto is False:
             self.clean_app_files(cleaned_sessions)
         if clean_pip is True:
             self.clean_pip_cache(auto=auto)
+        if clean_apps_cache is True:
+            self.clean_apps_cache()
 
     def _get_log_datetime(self, log_name) -> datetime:
         return datetime.strptime(log_name, "log_%Y-%m-%d_%H:%M:%S.txt")

agent/worker/constants.py

@@ -103,6 +103,7 @@ def TOKEN():
 _CONTAINER_NAME = "CONTAINER_NAME"
 _FORCE_CPU_ONLY = "FORCE_CPU_ONLY"
 _LOG_LEVEL = "LOG_LEVEL"
+_APPS_CACHE_DIR = "APPS_CACHE_DIR"
 
 _NET_CLIENT_DOCKER_IMAGE = "NET_CLIENT_DOCKER_IMAGE"
 _NET_SERVER_PORT = "NET_SERVER_PORT"
@@ -169,6 +170,7 @@ def TOKEN():
     _SLY_EXTRA_CA_CERTS_VOLUME_NAME: f"supervisely-agent-ca-certs-{TOKEN()[:8]}",
     _FORCE_CPU_ONLY: "false",
     _LOG_LEVEL: "INFO",
+    _APPS_CACHE_DIR: "/apps_cache",
 }
 
 
@@ -449,6 +451,14 @@ def AGENT_APPS_CACHE_DIR():
     return os.path.join(AGENT_ROOT_DIR(), "apps_cache")
 
 
+def APPS_CACHE_DIR():
+    """
+    Is used to access AGENT_APPS_CACHE_DIR from the app container.
+    default: /apps_cache
+    """
+    return read_optional_setting(_APPS_CACHE_DIR)
+
+
 def GITHUB_TOKEN():
     return read_optional_setting(_GITHUB_TOKEN)
 

agent/worker/task_app.py

@@ -138,7 +138,7 @@ def init_task_dir(self):
         self.dir_task_container = "/app"
 
         self.dir_task_src_container = os.path.join(self.dir_task_container, "repo")
-        self.dir_apps_cache_container = "/apps_cache"
+        self.dir_apps_cache_container = constants.APPS_CACHE_DIR()
         self.app_info = self.info["appInfo"]
 
     def download_or_get_repo(self):
@@ -690,6 +690,7 @@ def main_step_envs(self):
             "icon": self.app_config.get("icon", "https://cdn.supervise.ly/favicon.ico"),
             "PIP_ROOT_USER_ACTION": "ignore",
             "AGENT_ID": self.agent_id,
+            "APPS_CACHE_DIR": self.dir_apps_cache_container,
         }
 
         if "context.workspaceId" in envs: