Pull request #102: Add file caching

Merge in DEV/voyager from cacheFiles to master

* commit '29c97f508fc17722440986d71b83579c46036133':
  Add file caching

deploy/Installer.iss

@@ -1,5 +1,5 @@
 // ###########################
-#define Release "1.17.2"
+#define Release "1.17.3"
 // ###########################
 
 #define AppName "voyager"

docs/release-notes.md

@@ -1,5 +1,8 @@
 # Release notes
 
+### [1.17.3]
+- Add download caching. Cache can be cleared using `voyager cache clear`
+
 ### [1.17.2]
 - Fix the caching issue for remote repositories
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "voyager"
-version = "1.17.2"
+version = "1.17.3"
 description = "Package manager for C/C++ software"
 readme = "Readme.md"
 requires-python = "==3.10.*"
@@ -9,6 +9,7 @@ dependencies = [
     "dohq-artifactory==0.10.1",
     "jsonschema==4.23.0",
     "node-semver==0.7.0",
+    "platformdirs>=4.3.6",
     "voyager-plugin-list",
 ]
 
@@ -27,5 +28,5 @@ dev-dependencies = [
 ]
 
 [build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
+requires = ["hatchling==1.26.3"]
+build-backend = "hatchling.build"

voyager/artifactdownloader.py

@@ -20,11 +20,31 @@
 from artifactory import ArtifactoryPath
 import requests
 from semver import valid_range, max_satisfying
+from platformdirs import user_cache_dir
 import semver
 
 from .buildinfo import Package, BuildInfo
 from .configfile import ConfigFile
 from .lockfile import LockFileWriter
+from .utilities import md5hash
+
+class DownloadCachePaths:
+    @staticmethod
+    def _get_base_path() -> Path:
+        return Path(user_cache_dir("voyager", "ProdriveTechnologies"))
+
+    @staticmethod
+    def get_cache_path_for_package(package_dir) -> str:
+        return DownloadCachePaths._get_base_path() / package_dir / "voyager_package.tgz"
+
+    @staticmethod
+    def get_md5_path_for_package(package_dir) -> str:
+        return DownloadCachePaths._get_base_path() / package_dir / "voyager_package.tgz.md5"
+
+    @staticmethod
+    def clear_download_cache():
+        if os.path.exists(DownloadCachePaths._get_base_path()):
+            shutil.rmtree(DownloadCachePaths._get_base_path())
 
 class ArtifactDownloader:
     _download_dir = '.voyager'
@@ -40,13 +60,14 @@ def __init__(self, libraries: list, are_build_tools: bool, download_runtime_deps
 
     def clear_directory(self):
         try:
-            shutil.rmtree(self._download_dir)
+            if os.path.exists(self._download_dir):
+                shutil.rmtree(self._download_dir)
         except OSError as e:
             print(f'Error: {self._download_dir} : {e.strerror}')
 
     def make_directory(self):
         p = Path(self._download_dir)
-        p.mkdir()
+        p.mkdir(exist_ok=True)
 
     def download(self, build_info_combined):
         self._download(self.libraries, 0, build_info_combined, False)
@@ -287,9 +308,26 @@ def _find_download_extract_package(self, repo, library, version, output_dir, ove
             message = path.properties['deprecated']
             click.echo(click.style(f'DEPRECATED: {message} ', fg='yellow'), nl=False)
 
-        with path.open() as fd:
-            tar = tarfile.open(fileobj=fd)
-            tar.extractall(extract_dir)
+        localpath = DownloadCachePaths.get_cache_path_for_package(package_dir)
+        localmd5hash = None
+        localpathmd5 = DownloadCachePaths.get_md5_path_for_package(package_dir)
+        if localpathmd5.exists():
+            with open(localpathmd5, 'r') as file:
+                localmd5hash = file.read().rstrip()
+        else:
+            if localpath.exists():
+                localmd5hash = md5hash(localpath)
+                with open(localpathmd5, 'w') as file:
+                    file.write(localmd5hash)
+        if (not localpath.exists()) or (path.stat().md5 != localmd5hash):
+            localpath.parent.mkdir(exist_ok=True, parents=True)
+            with path.open() as fd, open(localpath, "wb") as out:
+                out.write(fd.read())
+        else:
+            click.echo(click.style(u'cached ', fg='green'), nl=False)
+
+        tar = tarfile.open(localpath)
+        tar.extractall(extract_dir)
 
         return extract_dir
 

voyager/utilities.py

@@ -14,6 +14,7 @@
 
 import sys
 import os
+import hashlib
 from pathlib import Path
 from typing import Optional
 
@@ -27,6 +28,16 @@ def resource_path(relative_path):
 
     return os.path.join(base_path, relative_path)
 
+def md5hash(filename):
+    h = hashlib.md5()
+    with open(filename,'rb') as f: 
+        while True:
+            data = f.read(8192)
+            if not data:
+                break
+            h.update(data)
+    return h.hexdigest()
+
 def solution_dot_voyager_path() -> Optional[str]:
     """ Get absolute path to the .voyager folder located in the solution directory 
     

voyager/voyager.py

@@ -30,7 +30,7 @@
 from .generators.packagelist import PackageListGenerator
 from .buildinfo import BuildInfo
 from .configfile import ConfigFile
-from .artifactdownloader import ArtifactDownloader
+from .artifactdownloader import ArtifactDownloader, DownloadCachePaths
 from .lockfile import LockFileWriter, LockFileReader
 from .voyagerpackagefile import VoyagerPackageFile
 from .cmakepackagefile import CMakePackageFile
@@ -42,7 +42,7 @@
 import voyager.doc as doc_server
 import voyager.package_update as package_updater
 
-VERSION = "1.17.2"
+VERSION = "1.17.3"
 SEARCH_RESULTS_FILE_NAME = Path("search_results.json")
 
 @click.group()
@@ -323,6 +323,16 @@ def check_update():
     """Check for available updates in the listed packages"""
     package_updater.execute_package_update()
 
+@cli.group()
+def cache():
+    """Commands for managing cache."""
+    pass
+
+@cache.command()
+def clear():
+    """Clears the cache."""
+    DownloadCachePaths.clear_download_cache()
+
 def main():
     print(f"Voyager version {VERSION}")