Merge pull request #140 from rjra2611/feature-issue-#54-add-python-3.…

…10-support

Feature issue #54 add python 3.10 support

.github/workflows/build.yml

@@ -9,10 +9,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-22.04, ubuntu-20.04, macos-12, macos-11, windows-latest]
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
-        exclude:
-          - os: ubuntu-22.04
-            python-version: '3.6'
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
       - name: Checkout
@@ -42,9 +39,9 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.6
+          python-version: 3.8
 
       - name: Install packaging dependencies
         run: pip install wheel

.github/workflows/regression-testing.yml

@@ -12,7 +12,7 @@ jobs:
       max-parallel: 1
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
       - name: Checkout

README.md

@@ -661,7 +661,7 @@ Usage: lean library add [OPTIONS] PROJECT NAME
   NAME must be the name of a NuGet package (for C# projects) or of a PyPI package (for Python projects).
 
   If --version is not given, the package is pinned to the latest compatible version. For C# projects, this is the latest
-  available version. For Python projects, this is the latest version compatible with Python 3.6 (which is what the
+  available version. For Python projects, this is the latest version compatible with Python 3.8 (which is what the
   Docker images use).
 
   Custom C# libraries are added to your project's .csproj file, which is then restored if dotnet is on your PATH and the
@@ -1244,7 +1244,7 @@ _See code: [lean/commands/whoami.py](lean/commands/whoami.py)_
 
 ## Development
 
-To work on the Lean CLI, clone the repository, enter an environment containing Python 3.6+ and run `pip install -r requirements.txt`. This command will install the required dependencies and installs the CLI in editable mode. This means you'll be able to edit the code and immediately see the results the next time you run `lean`.
+To work on the Lean CLI, clone the repository, enter an environment containing Python 3.7+ and run `pip install -r requirements.txt`. This command will install the required dependencies and installs the CLI in editable mode. This means you'll be able to edit the code and immediately see the results the next time you run `lean`.
 
 If you need to add dependencies, first update `setup.py` (if it is a production dependency) or `requirements.txt` (if it is a development dependency) and then re-run `pip install -r requirements.txt`.
 

announcements.json

@@ -1,5 +1,9 @@
 {
     "announcements": [
+        {
+            "date": "2022-08-26",
+            "message": "We've added support for python 3.10 and deprecated python version 3.6."
+        },
         {
             "date": "2022-05-23",
             "message": "We've added new json based approach of adding new modules to reduce the overhead and duplications."

lean/commands/create_project.py

@@ -113,7 +113,7 @@ def Subtract(self, a, b):
             "name": "python",
             "nbconvert_exporter": "python",
             "pygments_lexer": "ipython3",
-            "version": "3.6.8"
+            "version": "3.8.13"
         }
     },
     "nbformat": 4,

lean/commands/library/add.py

@@ -18,7 +18,7 @@
 from distutils.version import StrictVersion
 from pathlib import Path
 from typing import Any, Dict, Optional, Tuple
-
+from lean.constants import LEAN_STRICT_PYTHON_VERSION
 import click
 from dateutil.parser import isoparse
 from lxml import etree
@@ -158,8 +158,7 @@ def _get_pypi_package(name: str, version: Optional[str]) -> Tuple[str, str]:
     pypi_data = json.loads(response.text)
     name = pypi_data["info"]["name"]
 
-    required_python_version = "3.6.7" if platform.machine() in ["arm64", "aarch64"] else "3.6.8"
-    required_python_version = StrictVersion(required_python_version)
+    required_python_version = StrictVersion(LEAN_STRICT_PYTHON_VERSION)
 
     last_compatible_version = None
     last_compatible_version_upload_time = None
@@ -233,7 +232,7 @@ def _add_python(project_dir: Path, name: str, version: Optional[str], no_local:
 
     :param project_dir: the path to the project directory
     :param name: the name of the library to add
-    :param version: the version of the library to use, or None to pin to the latest version supporting Python 3.6
+    :param version: the version of the library to use, or None to pin to the latest version supporting Python 3.8
     :param no_local: whether installing the package in the local Python environment must be skipped
     """
     logger = container.logger()
@@ -275,7 +274,7 @@ def add(project: Path, name: str, version: Optional[str], no_local: bool) -> Non
 
     If --version is not given, the package is pinned to the latest compatible version.
     For C# projects, this is the latest available version.
-    For Python projects, this is the latest version compatible with Python 3.6 (which is what the Docker images use).
+    For Python projects, this is the latest version compatible with Python 3.8 (which is what the Docker images use).
 
     Custom C# libraries are added to your project's .csproj file,
     which is then restored if dotnet is on your PATH and the --no-local flag has not been given.

lean/components/docker/lean_runner.py

@@ -31,6 +31,7 @@
 from lean.components.util.temp_manager import TempManager
 from lean.components.util.xml_manager import XMLManager
 from lean.constants import MODULES_DIRECTORY, TERMINAL_LINK_PRODUCT_ID, LEAN_ROOT_PATH
+from lean.constants import DOCKER_PYTHON_SITE_PACKAGES_PATH
 from lean.models.docker import DockerImage
 from lean.models.utils import DebuggingMethod
 
@@ -401,7 +402,7 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any])
         # Mount a volume to the user packages directory so we don't install packages every time
         site_packages_volume = self._docker_manager.create_site_packages_volume(requirements_txt)
         run_options["volumes"][site_packages_volume] = {
-            "bind": "/root/.local/lib/python3.6/site-packages",
+            "bind": f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}",
             "mode": "rw"
         }
 
@@ -412,7 +413,7 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any])
         # We only need to do this if it hasn't already been done before for this site packages volume
         # To keep track of this we create a special file in the site packages directory after installation
         # If this file already exists we can skip pip install completely
-        marker_file = "/root/.local/lib/python3.6/site-packages/pip-install-done"
+        marker_file = f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}/pip-install-done"
         run_options["commands"].extend([
             f"! test -f {marker_file} && pip install --user --progress-bar off -r /requirements.txt",
             f"touch {marker_file}"

lean/components/util/http_client.py

@@ -16,6 +16,7 @@
 import requests
 
 from lean.components.util.logger import Logger
+from lean.models.errors import MoreInfoError
 
 
 class HTTPClient:
@@ -37,13 +38,7 @@ def get(self, url: str, **kwargs) -> requests.Response:
         :param kwargs: any kwargs to pass on to requests.get()
         :return: the response of the request
         """
-        self._log_request("GET", url, **kwargs)
-
-        raise_for_status = kwargs.pop("raise_for_status", True)
-        response = requests.get(url, **kwargs)
-
-        self._check_response(response, raise_for_status)
-        return response
+        return self.request("GET", url, **kwargs)
 
     def post(self, url: str, **kwargs) -> requests.Response:
         """A wrapper around requests.post().
@@ -54,13 +49,7 @@ def post(self, url: str, **kwargs) -> requests.Response:
         :param kwargs: any kwargs to pass on to requests.post()
         :return: the response of the request
         """
-        self._log_request("POST", url, **kwargs)
-
-        raise_for_status = kwargs.pop("raise_for_status", True)
-        response = requests.post(url, **kwargs)
-
-        self._check_response(response, raise_for_status)
-        return response
+        return self.request("POST", url, **kwargs)
 
     def request(self, method: str, url: str, **kwargs) -> requests.Response:
         """A wrapper around requests.request().
@@ -75,7 +64,15 @@ def request(self, method: str, url: str, **kwargs) -> requests.Response:
         self._log_request(method, url, **kwargs)
 
         raise_for_status = kwargs.pop("raise_for_status", True)
-        response = requests.request(method, url, **kwargs)
+        try:
+            response = requests.request(method, url, **kwargs)
+        except requests.exceptions.SSLError as e:
+            raise Exception(f"""
+Detected SSL error, this might be due to custom certificates in your environment or system trust store.
+A known limitation of the python requests implementation. 
+Please consider installing library https://pypi.org/project/python-certifi-win32/. 
+Related issue https://github.com/psf/requests/issues/2966
+    """.strip())
 
         self._check_response(response, raise_for_status)
         return response

lean/constants.py

@@ -16,6 +16,15 @@
 
 # Due to the way the filesystem is mocked in unit tests, values should not be Path instances.
 
+# The python version of docker image
+LEAN_PYTHON_VERSION = "3.8"
+
+# The strict python version of docker image
+LEAN_STRICT_PYTHON_VERSION =  f"{LEAN_PYTHON_VERSION}.13"
+
+# The path to the root python directory in docker image
+DOCKER_PYTHON_SITE_PACKAGES_PATH = f"/root/.local/lib/python{LEAN_PYTHON_VERSION}/site-packages"
+
 # The file in which general CLI configuration is stored
 LEAN_ROOT_PATH = "/Lean/Launcher/bin/Debug"
 

requirements.txt

@@ -4,7 +4,7 @@
 -e .
 
 wheel
-pytest==6.2.2
-pyfakefs==4.3.3
-responses==0.12.1
-lxml-stubs==0.1.1
+pytest==7.1.2
+pyfakefs==4.6.3
+responses==0.21.0
+lxml-stubs==0.4.0

setup.py

@@ -46,19 +46,17 @@ def get_stubs_version_range() -> str:
 
 # Production dependencies
 install_requires = [
-    "click~=8.0.4",
-    "requests~=2.27.1",
-    "json5~=0.9.8",
-    "docker~=5.0.3",
-    "rich~=9.10.0",
-    "dependency-injector~=4.39.1",
-    "pydantic~=1.8.2",
+    "click~=8.1.3",
+    "requests~=2.28.1",
+    "json5~=0.9.10",
+    "docker~=6.0.0",
+    "rich~=12.5.1",
+    "dependency-injector~=4.40.0",
+    "pydantic~=1.9.2",
     "python-dateutil~=2.8.2",
-    "lxml~=4.9.0",
+    "lxml~=4.9.1",
     "maskpass==0.3.6",
     "joblib~=1.1.0",
-    "python-certifi-win32~=1.6",
-    "pyshortcuts~=1.8.2",
     "wrapt~=1.14.1",
     "setuptools",
     f"quantconnect-stubs{get_stubs_version_range()}"
@@ -81,17 +79,17 @@ def get_stubs_version_range() -> str:
         "console_scripts": ["lean=lean.main:main"]
     },
     install_requires=install_requires,
-    python_requires=">= 3.6",
+    python_requires=">= 3.7",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
         "Intended Audience :: Financial and Insurance Industry",
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9"
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10"
     ],
     project_urls={
         "Documentation": "https://www.lean.io/docs/lean-cli/key-concepts/getting-started",

tests/commands/test_research.py

@@ -19,7 +19,7 @@
 from dependency_injector import providers
 
 from lean.commands import lean
-from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH
+from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH, LEAN_PYTHON_VERSION
 from lean.container import container
 from lean.models.docker import DockerImage
 from tests.test_helpers import create_fake_lean_cli_directory
@@ -164,10 +164,10 @@ def test_research_opens_browser_when_container_started(open) -> None:
     docker_manager.run_image.assert_called_once()
     args, kwargs = docker_manager.run_image.call_args
 
-    logs = """
+    logs = f"""
 [I 21:06:21.500 LabApp] Writing notebook server cookie secret to /root/.local/share/jupyter/runtime/notebook_cookie_secret
 [W 21:06:21.692 LabApp] All authentication is disabled.  Anyone who can connect to this server will be able to run code.
-[I 21:06:21.698 LabApp] JupyterLab extension loaded from /opt/miniconda3/lib/python3.6/site-packages/jupyterlab
+[I 21:06:21.698 LabApp] JupyterLab extension loaded from /opt/miniconda3/lib/python{LEAN_PYTHON_VERSION}/site-packages/jupyterlab
 [I 21:06:21.698 LabApp] JupyterLab application directory is /opt/miniconda3/share/jupyter/lab
 [I 21:06:21.700 LabApp] Serving notebooks from local directory: /Lean/Launcher/bin/Debug/Notebooks
 [I 21:06:21.700 LabApp] The Jupyter Notebook is running at:

tests/components/util/test_logger.py

@@ -67,7 +67,7 @@ def test_progress_creates_started_progress_instance(capsys: CaptureFixture) -> N
     logger = Logger()
     progress = logger.progress()
 
-    result = progress._started
+    result = progress.live._started
 
     progress.stop()
     assert result

tests/test_cli.py

@@ -204,7 +204,7 @@ def test_cli() -> None:
     assert (python_project_dir / "requirements.txt").is_file()
     assert f"altair==" in (python_project_dir / "requirements.txt").read_text(encoding="utf-8")
 
-    # Cannot add custom Python library incompatible with Python 3.6
+    # Cannot add custom Python library incompatible with lean.constants.LEAN_PYTHON_VERSION
     run_command(["lean", "library", "add", python_project_name, "PyS3DE"], cwd=test_dir, expected_return_code=1)
 
     # Cannot add custom Python library without version when it's not on PyPI
@@ -217,8 +217,8 @@ def test_cli() -> None:
                 cwd=test_dir,
                 expected_return_code=1)
 
-    # Cannot add custom Python library with version when version is incompatible with Python 3.6
-    run_command(["lean", "library", "add", python_project_name, "matplotlib", "--version", "3.4.2"],
+    # Cannot add custom Python library with version when version is incompatible with lean.constants.LEAN_PYTHON_VERSION
+    run_command(["lean", "library", "add", python_project_name, "matplotlib", "--version", "2.2.3"],
                 cwd=test_dir,
                 expected_return_code=1)