tools: Installing tools for given IDF_TARGET

Allow user to select specific ESP_TARGET while setting up ESD_IDF.
Only necessary tools for given target will be downloaded and installed.

Closes espressif#5113

docs/en/get-started/index.rst

@@ -140,7 +140,7 @@ If you have one of {IDF_TARGET_NAME} development boards listed below, you can cl
 
         ESP32-S3-DevKitC-1 <../hw-reference/esp32s3/user-guide-devkitc-1>
 
-        
+
 
 .. _get-started-step-by-step:
 
@@ -251,22 +251,33 @@ If you want to install the tools without the help of ESP-IDF Tools Installer, op
 .. code-block:: batch
 
     cd %userprofile%\esp\esp-idf
-    install.bat
+    install.bat {IDF_TARGET_PATH_NAME}
 
 or with Windows PowerShell
 
 .. code-block:: powershell
 
     cd ~/esp/esp-idf
-    ./install.ps1
+    ./install.ps1 {IDF_TARGET_PATH_NAME}
 
 Linux and macOS
 ~~~~~~~~~~~~~~~
 
 .. code-block:: bash
 
     cd ~/esp/esp-idf
-    ./install.sh
+    ./install.sh {IDF_TARGET_PATH_NAME}
+
+or with Fish shell
+
+.. code-block:: fish
+
+    cd ~/esp/esp-idf
+    ./install.fish {IDF_TARGET_PATH_NAME}
+
+.. note::
+    To install tools for multiple targets you can specify those targets at once. For example: ``./install.sh esp32,esp32c3,esp32s3``.
+    To install tools for all supported targets, run the script without specifying targets ``./install.sh`` or use ``./install.sh all``.
 
 Alternative File Downloads
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

install.bat

@@ -18,8 +18,11 @@ if not "%MISSING_REQUIREMENTS%" == "" goto :error_missing_requirements
 set IDF_PATH=%~dp0
 set IDF_PATH=%IDF_PATH:~0,-1%
 
+set TARGETS="all"
+if NOT "%1"=="" set TARGETS=%*
+
 echo Installing ESP-IDF tools
-python.exe %IDF_PATH%\tools\idf_tools.py install
+python.exe %IDF_PATH%\tools\idf_tools.py install --targets=%TARGETS%
 if %errorlevel% neq 0 goto :end
 
 echo Setting up Python environment

install.fish

@@ -7,8 +7,15 @@ set -x IDF_PATH $basedir
 echo "Detecting the Python interpreter"
 source "$IDF_PATH"/tools/detect_python.fish
 
+if not set -q argv[1]
+    set TARGETS "all"
+else
+    set TARGETS $argv[1]
+end
 echo "Installing ESP-IDF tools"
-"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install
+"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install --targets=$TARGETS
+    exit 1
+end
 
 echo "Installing Python environment and packages"
 "$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env

install.ps1

@@ -1,9 +1,14 @@
 #!/usr/bin/env pwsh
 $IDF_PATH = $PSScriptRoot
 
-
+if($args.count -eq 0){
+    $TARGETS = "all"
+}else
+{
+    $TARGETS = $args[0] -join ','
+}
 Write-Output "Installing ESP-IDF tools"
-Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install"
+Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install --targets=${TARGETS}"
 if ($LASTEXITCODE -ne 0)  { exit $LASTEXITCODE } # if error
 
 Write-Output "Setting up Python environment"

install.sh

@@ -8,8 +8,13 @@ export IDF_PATH=$(cd $(dirname $0); pwd)
 echo "Detecting the Python interpreter"
 . ${IDF_PATH}/tools/detect_python.sh
 
+if [ "$#" -eq 0 ]; then
+  TARGETS="all"
+else
+  TARGETS=$1
+fi
 echo "Installing ESP-IDF tools"
-${ESP_PYTHON} ${IDF_PATH}/tools/idf_tools.py install
+${ESP_PYTHON} ${IDF_PATH}/tools/idf_tools.py install --targets=${TARGETS}
 
 echo "Installing Python environment and packages"
 ${ESP_PYTHON} ${IDF_PATH}/tools/idf_tools.py install-python-env

tools/idf_tools.py

@@ -59,7 +59,7 @@
     print(e)
     raise SystemExit(1)
 
-from typing import IO, Callable, Optional, Tuple, Union  # noqa: F401
+from typing import IO, Any, Callable, Optional, Tuple, Union  # noqa: F401
 from urllib.error import ContentTooShortError
 from urllib.request import urlopen
 # the following is only for typing annotation
@@ -76,6 +76,7 @@ class WindowsError(OSError):  # type: ignore
 TOOLS_FILE = 'tools/tools.json'
 TOOLS_SCHEMA_FILE = 'tools/tools_schema.json'
 TOOLS_FILE_NEW = 'tools/tools.new.json'
+IDF_ENV_FILE = 'idf-env.json'
 TOOLS_FILE_VERSION = 1
 IDF_TOOLS_PATH_DEFAULT = os.path.join('~', '.espressif')
 UNKNOWN_VERSION = 'unknown'
@@ -482,7 +483,8 @@ def get_supported_platforms(self):  # type: () -> set[str]
                 'install',
                 'info_url',
                 'license',
-                'strip_container_dirs']
+                'strip_container_dirs',
+                'supported_targets']
 
 IDFToolOptions = namedtuple('IDFToolOptions', OPTIONS_LIST)  # type: ignore
 
@@ -493,9 +495,9 @@ class IDFTool(object):
     INSTALL_ON_REQUEST = 'on_request'
     INSTALL_NEVER = 'never'
 
-    def __init__(self, name, description, install, info_url, license, version_cmd, version_regex, version_regex_replace=None,
+    def __init__(self, name, description, install, info_url, license, version_cmd, version_regex, supported_targets, version_regex_replace=None,
                  strip_container_dirs=0):
-        # type: (str, str, str, str, str, list[str], str, Optional[str], int) -> None
+        # type: (str, str, str, str, str, list[str], str, list[str], Optional[str], int) -> None
         self.name = name
         self.description = description
         self.versions = OrderedDict()  # type: dict[str, IDFToolVersion]
@@ -504,7 +506,7 @@ def __init__(self, name, description, install, info_url, license, version_cmd, v
         if version_regex_replace is None:
             version_regex_replace = VERSION_REGEX_REPLACE_DEFAULT
         self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace,
-                                      [], OrderedDict(), install, info_url, license, strip_container_dirs)  # type: ignore
+                                      [], OrderedDict(), install, info_url, license, strip_container_dirs, supported_targets)  # type: ignore
         self.platform_overrides = []  # type: list[dict[str, str]]
         self._platform = CURRENT_PLATFORM
         self._update_current_options()
@@ -584,6 +586,9 @@ def check_version(self, extra_paths=None):  # type: (Optional[list[str]]) -> str
     def get_install_type(self):  # type: () -> Callable[[str], None]
         return self._current_options.install  # type: ignore
 
+    def get_supported_targets(self):  # type: ()  -> list[str]
+        return self._current_options.supported_targets  # type: ignore
+
     def compatible_with_platform(self):  # type: () -> bool
         return any([v.compatible_with_platform() for v in self.versions.values()])
 
@@ -797,9 +802,13 @@ def from_json(cls, tool_dict):  # type: (dict[str, Union[str, list[str], dict[st
         if type(overrides_list) is not list:
             raise RuntimeError('platform_overrides for tool %s is not a list' % tool_name)
 
+        supported_targets = tool_dict.get('supported_targets')
+        if not isinstance(supported_targets, list):
+            raise RuntimeError('supported_targets for tool %s is not a list of strings' % tool_name)
+
         # Create the object
         tool_obj = cls(tool_name, description, install, info_url, license,  # type: ignore
-                       version_cmd, version_regex, version_regex_replace,  # type: ignore
+                       version_cmd, version_regex, supported_targets, version_regex_replace,  # type: ignore
                        strip_container_dirs)  # type: ignore
 
         for path in export_paths:  # type: ignore
@@ -906,6 +915,7 @@ def to_json(self):  # type: ignore
             'license': self.options.license,
             'version_cmd': self.options.version_cmd,
             'version_regex': self.options.version_regex,
+            'supported_targets': self.options.supported_targets,
             'versions': versions_array,
         }
         if self.options.version_regex_replace != VERSION_REGEX_REPLACE_DEFAULT:
@@ -1013,6 +1023,98 @@ def get_python_env_path():  # type: () -> Tuple[str, str, str]
     return idf_python_env_path, idf_python_export_path, virtualenv_python
 
 
+def get_idf_env():  # type: () -> Any
+    try:
+        idf_env_file_path = os.path.join(global_idf_tools_path, IDF_ENV_FILE)  # type: ignore
+        with open(idf_env_file_path, 'r') as idf_env_file:
+            return json.load(idf_env_file)
+    except (IOError, OSError):
+        if not os.path.exists(idf_env_file_path):
+            warn('File {} was not found. '.format(idf_env_file_path))
+        else:
+            filename, ending = os.path.splitext(os.path.basename(idf_env_file_path))
+            warn('File {} can not be opened, renaming to {}'.format(idf_env_file_path,filename + '_failed' + ending))
+            os.rename(idf_env_file_path, os.path.join(os.path.dirname(idf_env_file_path), (filename + '_failed' + ending)))
+
+        info('Creating {}' .format(idf_env_file_path))
+        return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': {}}}}
+
+
+def export_targets_to_idf_env_json(targets):  # type: (list[str]) -> None
+    idf_env_json = get_idf_env()
+    targets = list(set(targets + get_user_defined_targets()))
+
+    for env in idf_env_json['idfInstalled']:
+        if env == idf_env_json['idfSelectedId']:
+            idf_env_json['idfInstalled'][env]['targets'] = targets
+            break
+
+    try:
+        with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w:  # type: ignore
+            json.dump(idf_env_json, w, indent=4)
+    except (IOError, OSError):
+        warn('File {} can not be created. '.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE)))  # type: ignore
+
+
+def clean_targets(targets_str):  # type: (str) -> list[str]
+    targets_from_tools_json = get_all_targets_from_tools_json()
+    invalid_targets = []
+
+    targets_str = targets_str.lower()
+    targets = targets_str.replace('-', '').split(',')
+    if targets != ['all']:
+        invalid_targets = [t for t in targets if t not in targets_from_tools_json]
+        if invalid_targets:
+            warn('Targets: "{}" are not supported. Only allowed options are: {}.'.format(', '.join(invalid_targets), ', '.join(targets_from_tools_json)))
+            raise SystemExit(1)
+        # removing duplicates
+        targets = list(set(targets))
+        export_targets_to_idf_env_json(targets)
+    else:
+        export_targets_to_idf_env_json(targets_from_tools_json)
+    return targets
+
+
+def get_user_defined_targets():  # type: () -> list[str]
+    try:
+        with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'r') as idf_env_file:  # type: ignore
+            idf_env_json = json.load(idf_env_file)
+    except OSError:
+        # warn('File {} was not found. Installing tools for all esp targets.'.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE)))  # type: ignore
+        return []
+
+    targets = []
+    for env in idf_env_json['idfInstalled']:
+        if env == idf_env_json['idfSelectedId']:
+            targets = idf_env_json['idfInstalled'][env]['targets']
+            break
+    return targets
+
+
+def get_all_targets_from_tools_json():  # type: () -> list[str]
+    tools_info = load_tools_info()
+    targets_from_tools_json = []  # type: list[str]
+
+    for _, v in tools_info.items():
+        targets_from_tools_json.extend(v.get_supported_targets())
+    # remove duplicates
+    targets_from_tools_json = list(set(targets_from_tools_json))
+    if 'all' in targets_from_tools_json:
+        targets_from_tools_json.remove('all')
+    return sorted(targets_from_tools_json)
+
+
+def filter_tools_info(tools_info):  # type: (OrderedDict[str, IDFTool]) -> OrderedDict[str,IDFTool]
+    targets = get_user_defined_targets()
+    if not targets:
+        return tools_info
+    else:
+        filtered_tools_spec = {k:v for k, v in tools_info.items() if
+                               (v.get_install_type() == IDFTool.INSTALL_ALWAYS or v.get_install_type() == IDFTool.INSTALL_ON_REQUEST) and
+                               (any(item in targets for item in v.get_supported_targets()) or v.get_supported_targets() == ['all'])}
+        return OrderedDict(filtered_tools_spec)
+
+
 def action_list(args):  # type: ignore
     tools_info = load_tools_info()
     for name, tool in tools_info.items():
@@ -1034,6 +1136,7 @@ def action_list(args):  # type: ignore
 
 def action_check(args):  # type: ignore
     tools_info = load_tools_info()
+    tools_info = filter_tools_info(tools_info)
     not_found_list = []
     info('Checking for installed tools...')
     for name, tool in tools_info.items():
@@ -1060,6 +1163,7 @@ def action_check(args):  # type: ignore
 
 def action_export(args):  # type: ignore
     tools_info = load_tools_info()
+    tools_info = filter_tools_info(tools_info)
     all_tools_found = True
     export_vars = {}
     paths_to_export = []
@@ -1241,6 +1345,10 @@ def apply_github_assets_option(tool_download_obj):  # type: ignore
 def action_download(args):  # type: ignore
     tools_info = load_tools_info()
     tools_spec = args.tools
+    targets = []  # type: list[str]
+    # Installing only single tools, no targets are specified.
+    if 'required' in tools_spec:
+        targets = clean_targets(args.targets)
 
     if args.platform not in PLATFORM_FROM_NAME:
         fatal('unknown platform: {}' % args.platform)
@@ -1253,8 +1361,17 @@ def action_download(args):  # type: ignore
         tools_info_for_platform[name] = tool_for_platform
 
     if not tools_spec or 'required' in tools_spec:
+        # Downloading tools for all ESP_targets required by the operating system.
         tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
+        # Filtering tools user defined list of ESP_targets
+        if 'all' not in targets:
+            def is_tool_selected(tool):  # type: (IDFTool) -> bool
+                supported_targets = tool.get_supported_targets()
+                return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
+            tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
         info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec)))
+
+    # Downloading tools for all ESP_targets (MacOS, Windows, Linux)
     elif 'all' in tools_spec:
         tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
         info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec)))
@@ -1288,9 +1405,24 @@ def action_download(args):  # type: ignore
 def action_install(args):  # type: ignore
     tools_info = load_tools_info()
     tools_spec = args.tools  # type: ignore
+    targets = []  # type: list[str]
+    # Installing only single tools, no targets are specified.
+    if 'required' in tools_spec:
+        targets = clean_targets(args.targets)
+        info('Selected targets are: {}' .format(', '.join(get_user_defined_targets())))
+
     if not tools_spec or 'required' in tools_spec:
+        # Installing tools for all ESP_targets required by the operating system.
         tools_spec = [k for k, v in tools_info.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
+        # Filtering tools user defined list of ESP_targets
+        if 'all' not in targets:
+            def is_tool_selected(tool):  # type: (IDFTool) -> bool
+                supported_targets = tool.get_supported_targets()
+                return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
+            tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
         info('Installing tools: {}'.format(', '.join(tools_spec)))
+
+    # Installing tools for all ESP_targets (MacOS, Windows, Linux)
     elif 'all' in tools_spec:
         tools_spec = [k for k, v in tools_info.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
         info('Installing tools: {}'.format(', '.join(tools_spec)))
@@ -1577,6 +1709,8 @@ def main(argv):  # type: (list[str]) -> None
                          'To install a specific version use <tool_name>@<version> syntax. ' +
                          'Use empty or \'required\' to install required tools, not optional ones. ' +
                          'Use \'all\' to install all tools, including the optional ones.')
+    install.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +
+                         ' It defaults to installing all supported targets.')
 
     download = subparsers.add_parser('download', help='Download the tools into the dist directory')
     download.add_argument('--platform', help='Platform to download the tools for')
@@ -1585,6 +1719,8 @@ def main(argv):  # type: (list[str]) -> None
                           'To download a specific version use <tool_name>@<version> syntax. ' +
                           'Use empty or \'required\' to download required tools, not optional ones. ' +
                           'Use \'all\' to download all tools, including the optional ones.')
+    download.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +
+                          ' It defaults to installing all supported targets.')
 
     if IDF_MAINTAINER:
         for subparser in [download, install]: