Merge pull request #175 from jhonabreul/feat-library-references-in-lo…

…cal-projects

Library projects for local development

README.md

@@ -682,11 +682,13 @@ Usage: lean library add [OPTIONS] PROJECT NAME
 
   PROJECT must be the path to the project.
 
-  NAME must be the name of a NuGet package (for C# projects) or of a PyPI package (for Python projects).
+  NAME must be either the name of a NuGet package (for C# projects), a PyPI package (for Python projects), or a path to
+  a Lean CLI library.
 
-  If --version is not given, the package is pinned to the latest compatible version. For C# projects, this is the latest
-  available version. For Python projects, this is the latest version compatible with Python 3.8 (which is what the
-  Docker images use).
+  If --version is not given, and the library is a NuGet or PyPI package the package, it is pinned to the latest
+  compatible version. For C# projects, this is the latest available version. For Python projects, this is the latest
+  version compatible with Python 3.8 (which is what the Docker images use). For Lean CLI library projects, this is
+  ignored.
 
   Custom C# libraries are added to your project's .csproj file, which is then restored if dotnet is on your PATH and the
   --no-local flag has not been given.
@@ -697,10 +699,12 @@ Usage: lean library add [OPTIONS] PROJECT NAME
   C# example usage:
   $ lean library add "My CSharp Project" Microsoft.ML
   $ lean library add "My CSharp Project" Microsoft.ML --version 1.5.5
+  $ lean library add "My CSharp Project" "Library/My CSharp Library"
 
   Python example usage:
   $ lean library add "My Python Project" tensorflow
   $ lean library add "My Python Project" tensorflow --version 2.5.0
+  $ lean library add "My Python Project" "Library/My Python Library"
 
 Options:
   --version TEXT  The version of the library to add (defaults to latest compatible version)
@@ -722,7 +726,8 @@ Usage: lean library remove [OPTIONS] PROJECT NAME
 
   PROJECT must be the path to the project directory.
 
-  NAME must be the name of the NuGet package (for C# projects) or of the PyPI package (for Python projects) to remove.
+  NAME must be either the name of the NuGet package (for C# projects), the PyPI package (for Python projects), or the
+  path to the Lean CLI library to remove.
 
   Custom C# libraries are removed from the project's .csproj file, which is then restored if dotnet is on your PATH and
   the --no-local flag has not been given.

lean/commands/backtest.py

@@ -355,10 +355,10 @@ def backtest(project: Path,
     # Set backtest name
     if backtest_name is not None and backtest_name != "":
         lean_config["backtest-name"] = backtest_name
-        
+
     if python_venv is not None and python_venv != "":
         lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}'
-    
+
     lean_runner = container.lean_runner()
     lean_runner.run_lean(lean_config,
                          "backtesting",

lean/commands/cloud/pull.py

@@ -11,12 +11,64 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional
+from pathlib import Path
+from typing import Optional, List
 
 import click
 
 from lean.click import LeanCommand
 from lean.container import container
+from lean.models.api import QCProject
+from lean.models.utils import LeanLibraryReference
+
+
+def _add_local_library_references_to_project(project: QCProject, cloud_libraries: List[QCProject]) -> None:
+    logger = container.logger()
+    library_manager = container.library_manager()
+
+    if len(cloud_libraries) > 0:
+        logger.info(f"Adding/updating local library references to project {project.name}")
+
+    cwd = Path.cwd()
+    project_dir = cwd / project.name
+    for i, library in enumerate(cloud_libraries, start=1):
+        logger.info(f"[{i}/{len(cloud_libraries)}] "
+                    f"Adding/updating local library {library.name} reference to project {project.name}")
+        library_manager.add_lean_library_to_project(project_dir, cwd / library.name, False)
+
+
+def _remove_local_library_references_from_project(project: QCProject, cloud_libraries: List[QCProject]) -> None:
+    logger = container.logger()
+    library_manager = container.library_manager()
+
+    project_dir = Path.cwd() / project.name
+    project_config = container.project_config_manager().get_project_config(project_dir)
+    local_libraries = project_config.get("libraries", [])
+    cloud_library_paths = [Path(library.name) for library in cloud_libraries]
+    libraries_to_remove = [LeanLibraryReference(**library_reference)
+                           for library_reference in local_libraries
+                           if Path(library_reference["path"]) not in cloud_library_paths]
+
+    if len(libraries_to_remove) > 0:
+        logger.info(f"Removing local library references from project {project.name}")
+
+    for i, library_reference in enumerate(libraries_to_remove, start=1):
+        logger.info(f"[{i}/{len(libraries_to_remove)}] "
+                    f"Removing local library {library_reference.name} reference from project {project.name}")
+        library_manager.remove_lean_library_from_project(project_dir, Path.cwd() / library_reference.path, False)
+
+
+def _update_local_library_references(projects: List[QCProject]) -> None:
+    for project in projects:
+        cloud_libraries = [library
+                           for library_id in project.libraries
+                           for library in projects if library.projectId == library_id]
+
+        # Add cloud library references to local config
+        _add_local_library_references_to_project(project, cloud_libraries)
+
+        # Remove library references locally if they were removed in the cloud
+        _remove_local_library_references_from_project(project, cloud_libraries)
 
 
 @click.command(cls=LeanCommand)
@@ -47,3 +99,5 @@ def pull(project: Optional[str], pull_bootcamp: bool) -> None:
 
     pull_manager = container.pull_manager()
     pull_manager.pull_projects(projects_to_pull)
+
+    _update_local_library_references(projects_to_pull)

lean/commands/cloud/push.py

@@ -12,13 +12,115 @@
 # limitations under the License.
 
 from pathlib import Path
-from typing import Optional
+from typing import Optional, List
 
 import click
 
 from lean.click import LeanCommand, PathParameter
 from lean.constants import PROJECT_CONFIG_FILE_NAME
 from lean.container import container
+from lean.models.api import QCProject
+from lean.models.utils import LeanLibraryReference
+
+
+def _get_cloud_project(project: Path, cloud_projects: List[QCProject]) -> QCProject:
+    lean_config_manager = container.lean_config_manager()
+    lean_cli_root_dir = lean_config_manager.get_cli_root_directory()
+    project_relative_path = project.relative_to(lean_cli_root_dir)
+    cloud_project = [cloud_project for cloud_project in cloud_projects
+                     if Path(cloud_project.name) == project_relative_path][0]
+
+    return cloud_project
+
+
+def _get_local_libraries_cloud_ids(project_dir: Path) -> List[int]:
+    project_config_manager = container.project_config_manager()
+    project_config = project_config_manager.get_project_config(project_dir)
+
+    lean_config_manager = container.lean_config_manager()
+    lean_cli_root_dir = lean_config_manager.get_cli_root_directory()
+
+    libraries_in_config = project_config.get("libraries", [])
+    library_paths = [lean_cli_root_dir / LeanLibraryReference(**library).path for library in libraries_in_config]
+
+    local_libraries_cloud_ids = [int(project_config_manager.get_project_config(path).get("cloud-id", None))
+                                 for path in library_paths]
+
+    return local_libraries_cloud_ids
+
+
+def _get_library_name(library_cloud_id: int, cloud_projects: List[QCProject]) -> str:
+    return [project.name for project in cloud_projects if project.projectId == library_cloud_id][0]
+
+
+def _add_new_libraries(project: QCProject,
+                       local_libraries_cloud_ids: List[int],
+                       cloud_projects: List[QCProject]) -> None:
+    logger = container.logger()
+    api_client = container.api_client()
+    libraries_to_add = [library_id for library_id in local_libraries_cloud_ids if library_id not in project.libraries]
+
+    if len(libraries_to_add) > 0:
+        logger.info(f"Adding libraries to project {project.name} in the cloud")
+
+    for i, library_cloud_id in enumerate(libraries_to_add, start=1):
+        library_name = _get_library_name(library_cloud_id, cloud_projects)
+        logger.info(f"[{i}/{len(libraries_to_add)}] "
+                    f"Adding library {library_name} to project {project.name} in the cloud")
+        api_client.projects.add_library(project.projectId, library_cloud_id)
+
+
+def _remove_outdated_libraries(project: QCProject,
+                               local_libraries_cloud_ids: List[int],
+                               cloud_projects: List[QCProject]) -> None:
+    logger = container.logger()
+    api_client = container.api_client()
+    libraries_to_remove = [library_id for library_id in project.libraries
+                           if library_id not in local_libraries_cloud_ids]
+
+    if len(libraries_to_remove) > 0:
+        logger.info(f"Removing libraries from project {project.name} in the cloud")
+
+    for i, library_cloud_id in enumerate(libraries_to_remove, start=1):
+        library_name = _get_library_name(library_cloud_id, cloud_projects)
+        logger.info(f"[{i}/{len(libraries_to_remove)}] "
+                    f"Removing library {library_name} from project {project.name} in the cloud")
+        api_client.projects.delete_library(project.projectId, library_cloud_id)
+
+
+def _update_cloud_library_references(projects: List[Path]) -> None:
+    api_client = container.api_client()
+    cloud_projects = api_client.projects.get_all()
+
+    for project in projects:
+        cloud_project = _get_cloud_project(project, cloud_projects)
+        local_libraries_cloud_ids = _get_local_libraries_cloud_ids(project)
+
+        _add_new_libraries(cloud_project, local_libraries_cloud_ids, cloud_projects)
+        _remove_outdated_libraries(cloud_project, local_libraries_cloud_ids, cloud_projects)
+
+
+def _get_libraries_to_push(project_dir: Path, seen_projects: List[Path] = None) -> List[Path]:
+    if seen_projects is None:
+        seen_projects = [project_dir]
+
+    project_config_manager = container.project_config_manager()
+    project_config = project_config_manager.get_project_config(project_dir)
+    libraries_in_config = project_config.get("libraries", [])
+    libraries = [LeanLibraryReference(**library).path.expanduser().resolve() for library in libraries_in_config]
+
+    referenced_libraries = []
+    for library_path in libraries:
+        # Avoid infinite recursion
+        if library_path in seen_projects:
+            continue
+
+        seen_projects.append(library_path)
+        referenced_libraries.extend(_get_libraries_to_push(library_path, seen_projects))
+
+    libraries.extend(referenced_libraries)
+
+    return list(dict.fromkeys(libraries))
 
 
 @click.command(cls=LeanCommand)
@@ -39,12 +141,15 @@ def push(project: Optional[Path], organization_id: Optional[str]) -> None:
     # Parse which projects need to be pushed
     if project is not None:
         project_config_manager = container.project_config_manager()
-        if not project_config_manager.get_project_config(project).file.exists():
+        project_config = project_config_manager.get_project_config(project)
+        if not project_config.file.exists():
             raise RuntimeError(f"'{project}' is not a Lean project")
 
-        projects_to_push = [project]
+        projects_to_push = [project, *_get_libraries_to_push(project)]
     else:
         projects_to_push = [p.parent for p in Path.cwd().rglob(PROJECT_CONFIG_FILE_NAME)]
 
     push_manager = container.push_manager()
     push_manager.push_projects(projects_to_push, organization_id)
+
+    _update_cloud_library_references(projects_to_push)

lean/commands/library/add.py

@@ -12,7 +12,6 @@
 # limitations under the License.
 
 import json
-import platform
 import shutil
 import subprocess
 from distutils.version import StrictVersion
@@ -83,40 +82,30 @@ def _add_csharp_package_to_csproj(csproj_file: Path, name: str, version: str) ->
     csproj_file.write_text(xml_manager.to_string(csproj_tree), encoding="utf-8")
 
 
-def _add_csharp(project_dir: Path, name: str, version: Optional[str], no_local: bool) -> None:
-    """Adds a custom C# library to a C# project.
+def _add_nuget_package_to_csharp_project(project_dir: Path, name: str, version: Optional[str], no_local: bool) -> None:
+    """Adds a NuGet package to the project in the given directory.
 
-    Adds the library to the project's .csproj file, and restores the project is dotnet is on the user's PATH.
+    Adds the library to the project's .csproj file, and restores the project if dotnet is on the user's PATH.
 
     :param project_dir: the path to the project directory
     :param name: the name of the library to add
     :param version: the version of the library to use, or None to pin to the latest version
     :param no_local: whether restoring the packages locally must be skipped
     """
     logger = container.logger()
-    path_manager = container.path_manager()
 
     if version is None:
         logger.info("Retrieving latest available version from NuGet")
         name, version = _get_nuget_package(name)
 
-    csproj_file = next(p for p in project_dir.iterdir() if p.name.endswith(".csproj"))
+    project_manager = container.project_manager()
+    csproj_file = project_manager.get_csproj_file_path(project_dir)
+    path_manager = container.path_manager()
     logger.info(f"Adding {name} {version} to '{path_manager.get_relative_path(csproj_file)}'")
 
     original_csproj_content = csproj_file.read_text(encoding="utf-8")
     _add_csharp_package_to_csproj(csproj_file, name, version)
-
-    if not no_local and shutil.which("dotnet") is not None:
-        logger.info(
-            f"Restoring packages in '{path_manager.get_relative_path(project_dir)}' to provide local autocomplete")
-
-        process = subprocess.run(["dotnet", "restore", str(csproj_file)], cwd=project_dir)
-
-        if process.returncode != 0:
-            logger.warn(f"Reverting the changes to '{path_manager.get_relative_path(csproj_file)}'")
-            csproj_file.write_text(original_csproj_content, encoding="utf-8")
-
-            raise RuntimeError("Something went wrong while restoring packages, see the logs above for more information")
+    project_manager.try_restore_csharp_project(csproj_file, original_csproj_content, no_local)
 
 
 def _is_pypi_file_compatible(file: Dict[str, Any], required_python_version: StrictVersion) -> bool:
@@ -224,7 +213,7 @@ def _add_python_package_to_requirements(requirements_file: Path, name: str, vers
     requirements_file.write_text(new_content, encoding="utf-8")
 
 
-def _add_python(project_dir: Path, name: str, version: Optional[str], no_local: bool) -> None:
+def _add_pypi_package_to_python_project(project_dir: Path, name: str, version: Optional[str], no_local: bool) -> None:
     """Adds a custom Python library to a Python project.
 
     Adds the library to the project's requirements.txt file,
@@ -256,8 +245,8 @@ def _add_python(project_dir: Path, name: str, version: Optional[str], no_local:
         process = subprocess.run(["pip", "install", f"{name}=={version}"])
 
         if process.returncode != 0:
-            raise RuntimeError(
-                f"Something went wrong while installing {name} {version} locally, see the logs above for more information")
+            raise RuntimeError(f"Something went wrong while installing {name} {version} "
+                               "locally, see the logs above for more information")
 
 
 @click.command(cls=LeanCommand)
@@ -270,11 +259,14 @@ def add(project: Path, name: str, version: Optional[str], no_local: bool) -> Non
 
     PROJECT must be the path to the project.
 
-    NAME must be the name of a NuGet package (for C# projects) or of a PyPI package (for Python projects).
+    NAME must be either the name of a NuGet package (for C# projects), a PyPI package (for Python projects),
+    or a path to a Lean CLI library.
 
-    If --version is not given, the package is pinned to the latest compatible version.
+    If --version is not given, and the library is a NuGet or PyPI package the package, it is pinned to the latest
+    compatible version.
     For C# projects, this is the latest available version.
     For Python projects, this is the latest version compatible with Python 3.8 (which is what the Docker images use).
+    For Lean CLI library projects, this is ignored.
 
     Custom C# libraries are added to your project's .csproj file,
     which is then restored if dotnet is on your PATH and the --no-local flag has not been given.
@@ -286,20 +278,34 @@ def add(project: Path, name: str, version: Optional[str], no_local: bool) -> Non
     C# example usage:
     $ lean library add "My CSharp Project" Microsoft.ML
     $ lean library add "My CSharp Project" Microsoft.ML --version 1.5.5
+    $ lean library add "My CSharp Project" "Library/My CSharp Library"
 
     \b
     Python example usage:
     $ lean library add "My Python Project" tensorflow
     $ lean library add "My Python Project" tensorflow --version 2.5.0
+    $ lean library add "My Python Project" "Library/My Python Library"
     """
+    logger = container.logger()
     project_config = container.project_config_manager().get_project_config(project)
     project_language = project_config.get("algorithm-language", None)
 
     if project_language is None:
         raise MoreInfoError(f"{project} is not a Lean CLI project",
                             "https://www.lean.io/docs/v2/lean-cli/projects/project-management#02-Create-Projects")
 
-    if project_language == "CSharp":
-        _add_csharp(project, name, version, no_local)
+    library_manager = container.library_manager()
+    library_dir = Path(name).expanduser().resolve()
+
+    if library_manager.is_lean_library(library_dir):
+        logger.info(f"Adding Lean CLI library {library_dir} to project {project}")
+        if project_language == "CSharp":
+            library_manager.add_lean_library_to_csharp_project(project, library_dir, no_local)
+        else:
+            library_manager.add_lean_library_to_python_project(project, library_dir)
     else:
-        _add_python(project, name, version, no_local)
+        logger.info(f"Adding package {name} to project {project}")
+        if project_language == "CSharp":
+            _add_nuget_package_to_csharp_project(project, name, version, no_local)
+        else:
+            _add_pypi_package_to_python_project(project, name, version, no_local)