From b2c3c504e087cb3f8d6431d951b891cccf98cd74 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 15 Nov 2024 02:09:07 -0500 Subject: [PATCH 1/3] feat: adding a schema command Now running this passes: uvx check-jsonschema --schemafile src/tox/tox.schema.json tox.toml Signed-off-by: Henry Schreiner --- pyproject.toml | 3 + src/tox/config/main.py | 3 + src/tox/config/sets.py | 8 +- src/tox/plugin/manager.py | 2 + src/tox/session/cmd/schema.py | 174 ++++++++++++++ src/tox/tox.schema.json | 439 ++++++++++++++++++++++++++++++++++ 6 files changed, 628 insertions(+), 1 deletion(-) create mode 100644 src/tox/session/cmd/schema.py create mode 100644 src/tox/tox.schema.json diff --git a/pyproject.toml b/pyproject.toml index f200a6d2d..f4b011cc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -253,3 +253,6 @@ overrides = [ "virtualenv.*", ], ignore_missing_imports = true }, ] + +[tool.uv] +environments = [ "python_version >= '3.10'" ] diff --git a/src/tox/config/main.py b/src/tox/config/main.py index 6e734191b..8c9d45737 100644 --- a/src/tox/config/main.py +++ b/src/tox/config/main.py @@ -123,6 +123,9 @@ def core(self) -> CoreConfigSet: self._core_set = core return core + def all_section_configs(self) -> Iterator[tuple[tuple[str, str, str], ConfigSet]]: + yield from self._key_to_conf_set.items() + def get_section_config( self, section: Section, diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py index b78ff84cb..7db61d1a3 100644 --- a/src/tox/config/sets.py +++ b/src/tox/config/sets.py @@ -3,7 +3,7 @@ import sys from abc import ABC, abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Iterator, Mapping, Sequence, TypeVar, cast +from typing import TYPE_CHECKING, Any, Callable, Generator, Iterator, Mapping, Sequence, TypeVar, cast from .of_type import ConfigConstantDefinition, ConfigDefinition, ConfigDynamicDefinition, ConfigLoadArgs from .set_env import SetEnv @@ -33,6 +33,12 @@ def __init__(self, conf: Config, section: Section, env_name: str | None) -> None self._final = False self.register_config() + def get_configs(self) -> Generator[ConfigDefinition[Any], None, None]: + """:return: a mapping of config keys to their definitions""" + for k, v in self._defined.items(): + if k == next(iter(v.keys)): + yield v + @abstractmethod def register_config(self) -> None: raise NotImplementedError diff --git a/src/tox/plugin/manager.py b/src/tox/plugin/manager.py index 58a563043..2b1366bfc 100644 --- a/src/tox/plugin/manager.py +++ b/src/tox/plugin/manager.py @@ -42,6 +42,7 @@ def _register_plugins(self, inline: ModuleType | None) -> None: legacy, list_env, quickstart, + schema, show_config, version_flag, ) @@ -60,6 +61,7 @@ def _register_plugins(self, inline: ModuleType | None) -> None: exec_, quickstart, show_config, + schema, devenv, list_env, depends, diff --git a/src/tox/session/cmd/schema.py b/src/tox/session/cmd/schema.py new file mode 100644 index 000000000..dd32b4dc2 --- /dev/null +++ b/src/tox/session/cmd/schema.py @@ -0,0 +1,174 @@ +"""Generate schema for tox configuration, respecting the current plugins.""" + +from __future__ import annotations + +import json +import sys +import typing +from pathlib import Path +from types import NoneType +from typing import TYPE_CHECKING + +import packaging.requirements +import packaging.version + +import tox.config.set_env +import tox.config.types +import tox.tox_env.python.pip.req_file +from tox.plugin import impl + +if TYPE_CHECKING: + from tox.config.cli.parser import ToxParser + from tox.config.sets import ConfigSet + from tox.session.state import State + + +@impl +def tox_add_option(parser: ToxParser) -> None: + our = parser.add_command("schema", [], "Generate schema for tox configuration", gen_schema) + our.add_argument("--strict", action="store_true", help="Disallow extra properties in configuration") + + +def _process_type(of_type: typing.Any) -> dict[str, typing.Any]: # noqa: C901, PLR0911 + if of_type in { + Path, + str, + packaging.version.Version, + packaging.requirements.Requirement, + tox.tox_env.python.pip.req_file.PythonDeps, + }: + return {"type": "string"} + if typing.get_origin(of_type) is typing.Union: + types = [x for x in typing.get_args(of_type) if x is not NoneType] + if len(types) == 1: + return _process_type(types[0]) + msg = f"Union types are not supported: {of_type}" + raise ValueError(msg) + if of_type is bool: + return {"type": "boolean"} + if of_type is float: + return {"type": "number"} + if typing.get_origin(of_type) is typing.Literal: + return {"enum": list(typing.get_args(of_type))} + if of_type in {tox.config.types.Command, tox.config.types.EnvList}: + return {"type": "array", "items": {"$ref": "#/definitions/subs"}} + if typing.get_origin(of_type) in {list, set}: + if typing.get_args(of_type)[0] in {str, packaging.requirements.Requirement}: + return {"type": "array", "items": {"$ref": "#/definitions/subs"}} + if typing.get_args(of_type)[0] is tox.config.types.Command: + return {"type": "array", "items": _process_type(typing.get_args(of_type)[0])} + msg = f"Unknown list type: {of_type}" + raise ValueError(msg) + if of_type is tox.config.set_env.SetEnv: + return { + "type": "object", + "additionalProperties": {"$ref": "#/definitions/subs"}, + } + if typing.get_origin(of_type) is dict: + return { + "type": "object", + "additionalProperties": {**_process_type(typing.get_args(of_type)[1])}, + } + msg = f"Unknown type: {of_type}" + raise ValueError(msg) + + +def _get_schema(conf: ConfigSet, path: str) -> dict[str, dict[str, typing.Any]]: + properties = {} + for x in conf.get_configs(): + name, *aliases = x.keys + of_type = getattr(x, "of_type", None) + if of_type is None: + continue + desc = getattr(x, "desc", None) + try: + properties[name] = {**_process_type(of_type), "description": desc} + except ValueError: + print(name, "has unrecoginsed type:", of_type, file=sys.stderr) # noqa: T201 + for alias in aliases: + properties[alias] = {"$ref": f"{path}/{name}"} + return properties + + +def gen_schema(state: State) -> int: + core = state.conf.core + strict = state.conf.options.strict + + # Accessing this adds extra stuff to core, so we need to do it first + env_properties = _get_schema(state.envs["3.13"].conf, path="#/properties/env_run_base/properties") + + properties = _get_schema(core, path="#/properties") + + sections = { + key: conf + for s, conf in state.conf.all_section_configs() + if (key := s[0].split(".")[0]) not in {"env_run_base", "env_pkg_base", "env"} + } + for key, conf in sections.items(): + properties[key] = { + "type": "object", + "additionalProperties": not strict, + "properties": _get_schema(conf, path=f"#/properties/{key}/properties"), + } + + json_schema = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://github.com/tox-dev/tox/blob/main/src/tox/util/tox.schema.json", + "type": "object", + "properties": { + **properties, + "env_run_base": { + "type": "object", + "properties": env_properties, + "additionalProperties": not strict, + }, + "env_pkg_base": { + "$ref": "#/properties/env_run_base", + "additionalProperties": not strict, + }, + "env": {"type": "object", "patternProperties": {"^.*$": {"$ref": "#/properties/env_run_base"}}}, + "legacy_tox_ini": {"type": "string"}, + }, + "additionalProperties": not strict, + "definitions": { + "subs": { + "anyOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "replace": {"type": "string"}, + "name": {"type": "string"}, + "default": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"$ref": "#/definitions/subs"}}, + ] + }, + "extend": {"type": "boolean"}, + }, + "required": ["replace"], + "additionalProperties": False, + }, + { + "type": "object", + "properties": { + "replace": {"type": "string"}, + "of": {"type": "array", "items": {"type": "string"}}, + "default": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"$ref": "#/definitions/subs"}}, + ] + }, + "extend": {"type": "boolean"}, + }, + "required": ["replace", "of"], + "additionalProperties": False, + }, + ], + }, + }, + } + print(json.dumps(json_schema, indent=2)) # noqa: T201 + return 0 diff --git a/src/tox/tox.schema.json b/src/tox/tox.schema.json new file mode 100644 index 000000000..1a2c9d26f --- /dev/null +++ b/src/tox/tox.schema.json @@ -0,0 +1,439 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://github.com/tox-dev/tox/blob/main/src/tox/util/tox.schema.json", + "type": "object", + "properties": { + "tox_root": { + "type": "string", + "description": "the root directory (where the configuration file is found)" + }, + "toxinidir": { + "$ref": "#/properties/tox_root" + }, + "work_dir": { + "type": "string", + "description": "working directory" + }, + "toxworkdir": { + "$ref": "#/properties/work_dir" + }, + "temp_dir": { + "type": "string", + "description": "a folder for temporary files (is not cleaned at start)" + }, + "env_list": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "define environments to automatically run" + }, + "envlist": { + "$ref": "#/properties/env_list" + }, + "base": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "inherit missing keys from these sections" + }, + "min_version": { + "type": "string", + "description": "Define the minimal tox version required to run" + }, + "minversion": { + "$ref": "#/properties/min_version" + }, + "provision_tox_env": { + "type": "string", + "description": "Name of the virtual environment used to provision a tox." + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "Name of the virtual environment used to provision a tox." + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + } + }, + "description": "core labels" + }, + "ignore_base_python_conflict": { + "type": "boolean", + "description": "do not raise error if the environment name conflicts with base python" + }, + "ignore_basepython_conflict": { + "$ref": "#/properties/ignore_base_python_conflict" + }, + "skip_missing_interpreters": { + "type": "boolean", + "description": "skip running missing interpreters" + }, + "no_package": { + "type": "boolean", + "description": "is there any packaging involved in this project" + }, + "skipsdist": { + "$ref": "#/properties/no_package" + }, + "package_env": { + "type": "string", + "description": "tox environment used to package" + }, + "isolated_build_env": { + "$ref": "#/properties/package_env" + }, + "package_root": { + "type": "string", + "description": "indicates where the packaging root file exists (historically setup.py file or pyproject.toml now)" + }, + "setupdir": { + "$ref": "#/properties/package_root" + }, + "env_run_base": { + "type": "object", + "properties": { + "set_env": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/subs" + }, + "description": "environment variables to set when running commands in the tox environment" + }, + "setenv": { + "$ref": "#/properties/env_run_base/properties/set_env" + }, + "base": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "inherit missing keys from these sections" + }, + "runner": { + "type": "string", + "description": "the tox execute used to evaluate this environment" + }, + "description": { + "type": "string", + "description": "description attached to the tox environment" + }, + "depends": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "tox environments that this environment depends on (must be run after those)" + }, + "labels": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "labels attached to the tox environment" + }, + "env_dir": { + "type": "string", + "description": "directory assigned to the tox environment" + }, + "envdir": { + "$ref": "#/properties/env_run_base/properties/env_dir" + }, + "env_tmp_dir": { + "type": "string", + "description": "a folder that is always reset at the start of the run" + }, + "envtmpdir": { + "$ref": "#/properties/env_run_base/properties/env_tmp_dir" + }, + "env_log_dir": { + "type": "string", + "description": "a folder for logging where tox will put logs of tool invocation" + }, + "envlogdir": { + "$ref": "#/properties/env_run_base/properties/env_log_dir" + }, + "suicide_timeout": { + "type": "number", + "description": "timeout to allow process to exit before sending SIGINT" + }, + "interrupt_timeout": { + "type": "number", + "description": "timeout before sending SIGTERM after SIGINT" + }, + "terminate_timeout": { + "type": "number", + "description": "timeout before sending SIGKILL after SIGTERM" + }, + "platform": { + "type": "string", + "description": "run on platforms that match this regular expression (empty means any platform)" + }, + "pass_env": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "environment variables to pass on to the tox environment" + }, + "passenv": { + "$ref": "#/properties/env_run_base/properties/pass_env" + }, + "parallel_show_output": { + "type": "boolean", + "description": "if set to True the content of the output will always be shown when running in parallel mode" + }, + "recreate": { + "type": "boolean", + "description": "always recreate virtual environment if this option is true, otherwise leave it up to tox" + }, + "allowlist_externals": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "external command glob to allow calling" + }, + "list_dependencies_command": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "command used to list installed packages" + }, + "pip_pre": { + "type": "boolean", + "description": "install the latest available pre-release (alpha/beta/rc) of dependencies without a specified version" + }, + "install_command": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "command used to install packages" + }, + "constrain_package_deps": { + "type": "boolean", + "description": "If true, apply constraints during install_package_deps." + }, + "use_frozen_constraints": { + "type": "boolean", + "description": "Use the exact versions of installed deps as constraints, otherwise use the listed deps." + }, + "commands_pre": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + } + }, + "description": "the commands to be called before testing" + }, + "commands": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + } + }, + "description": "the commands to be called for testing" + }, + "commands_post": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + } + }, + "description": "the commands to be called after testing" + }, + "change_dir": { + "type": "string", + "description": "change to this working directory when executing the test command" + }, + "changedir": { + "$ref": "#/properties/env_run_base/properties/change_dir" + }, + "args_are_paths": { + "type": "boolean", + "description": "if True rewrite relative posargs paths from cwd to change_dir" + }, + "ignore_errors": { + "type": "boolean", + "description": "when executing the commands keep going even if a sub-command exits with non-zero exit code" + }, + "ignore_outcome": { + "type": "boolean", + "description": "if set to true a failing result of this testenv will not make tox fail (instead just warn)" + }, + "base_python": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "environment identifier for python, first one found wins" + }, + "basepython": { + "$ref": "#/properties/env_run_base/properties/base_python" + }, + "deps": { + "type": "string", + "description": "Name of the python dependencies as specified by PEP-440" + }, + "dependency_groups": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "dependency groups to install of the target package" + }, + "system_site_packages": { + "type": "boolean", + "description": "create virtual environments that also have access to globally installed packages." + }, + "sitepackages": { + "$ref": "#/properties/env_run_base/properties/system_site_packages" + }, + "always_copy": { + "type": "boolean", + "description": "force virtualenv to always copy rather than symlink" + }, + "alwayscopy": { + "$ref": "#/properties/env_run_base/properties/always_copy" + }, + "download": { + "type": "boolean", + "description": "true if you want virtualenv to upgrade pip/wheel/setuptools to the latest version" + }, + "skip_install": { + "type": "boolean", + "description": "skip installation" + }, + "use_develop": { + "type": "boolean", + "description": "use develop mode" + }, + "usedevelop": { + "$ref": "#/properties/env_run_base/properties/use_develop" + }, + "package": { + "type": "string", + "description": "package installation mode - wheel | sdist | editable | editable-legacy | skip | external " + }, + "extras": { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + }, + "description": "extras to install of the target package" + }, + "package_env": { + "type": "string", + "description": "tox environment used to package" + }, + "wheel_build_env": { + "type": "string", + "description": "wheel tag to use for building applications" + } + }, + "additionalProperties": true + }, + "env_pkg_base": { + "$ref": "#/properties/env_run_base", + "additionalProperties": true + }, + "env": { + "type": "object", + "patternProperties": { + "^.*$": { + "$ref": "#/properties/env_run_base" + } + } + }, + "legacy_tox_ini": { + "type": "string" + } + }, + "additionalProperties": true, + "definitions": { + "subs": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "replace": { + "type": "string" + }, + "name": { + "type": "string" + }, + "default": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + } + } + ] + }, + "extend": { + "type": "boolean" + } + }, + "required": ["replace"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "replace": { + "type": "string" + }, + "of": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/subs" + } + } + ] + }, + "extend": { + "type": "boolean" + } + }, + "required": ["replace", "of"], + "additionalProperties": false + } + ] + } + } +} From 012325126f5b52581e44c30df27888fe7c7eb3a4 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 20 Nov 2024 00:23:56 -0500 Subject: [PATCH 2/3] refactor: leave access private Signed-off-by: Henry Schreiner --- pyproject.toml | 3 --- src/tox/config/main.py | 3 --- src/tox/session/cmd/schema.py | 5 ++++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f4b011cc2..f200a6d2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -253,6 +253,3 @@ overrides = [ "virtualenv.*", ], ignore_missing_imports = true }, ] - -[tool.uv] -environments = [ "python_version >= '3.10'" ] diff --git a/src/tox/config/main.py b/src/tox/config/main.py index 8c9d45737..6e734191b 100644 --- a/src/tox/config/main.py +++ b/src/tox/config/main.py @@ -123,9 +123,6 @@ def core(self) -> CoreConfigSet: self._core_set = core return core - def all_section_configs(self) -> Iterator[tuple[tuple[str, str, str], ConfigSet]]: - yield from self._key_to_conf_set.items() - def get_section_config( self, section: Section, diff --git a/src/tox/session/cmd/schema.py b/src/tox/session/cmd/schema.py index dd32b4dc2..3a4af420e 100644 --- a/src/tox/session/cmd/schema.py +++ b/src/tox/session/cmd/schema.py @@ -99,9 +99,12 @@ def gen_schema(state: State) -> int: properties = _get_schema(core, path="#/properties") + # This accesses plugins that register new sections (like tox-gh) + # Accessing a private member since this is not exposed yet and the + # interface includes the internal storage tuple sections = { key: conf - for s, conf in state.conf.all_section_configs() + for s, conf in state.conf._key_to_conf_set.items() # noqa: SLF001 if (key := s[0].split(".")[0]) not in {"env_run_base", "env_pkg_base", "env"} } for key, conf in sections.items(): From 602de5b822c029a518750ccdf0e1e84e64590890 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 20 Nov 2024 00:33:54 -0500 Subject: [PATCH 3/3] fix: changelog and update test list Signed-off-by: Henry Schreiner --- docs/changelog/3446.feature.rst | 3 +++ src/tox/session/cmd/schema.py | 5 ++--- tests/config/cli/conftest.py | 2 ++ tests/session/cmd/test_schema.py | 19 +++++++++++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/3446.feature.rst create mode 100644 tests/session/cmd/test_schema.py diff --git a/docs/changelog/3446.feature.rst b/docs/changelog/3446.feature.rst new file mode 100644 index 000000000..6edb4c852 --- /dev/null +++ b/docs/changelog/3446.feature.rst @@ -0,0 +1,3 @@ +Add a ``schema`` command to produce a JSON Schema for tox and the current plugins. + +- by :user:`henryiii` diff --git a/src/tox/session/cmd/schema.py b/src/tox/session/cmd/schema.py index 3a4af420e..ee708756d 100644 --- a/src/tox/session/cmd/schema.py +++ b/src/tox/session/cmd/schema.py @@ -6,7 +6,6 @@ import sys import typing from pathlib import Path -from types import NoneType from typing import TYPE_CHECKING import packaging.requirements @@ -39,7 +38,7 @@ def _process_type(of_type: typing.Any) -> dict[str, typing.Any]: # noqa: C901, }: return {"type": "string"} if typing.get_origin(of_type) is typing.Union: - types = [x for x in typing.get_args(of_type) if x is not NoneType] + types = [x for x in typing.get_args(of_type) if x is not type(None)] if len(types) == 1: return _process_type(types[0]) msg = f"Union types are not supported: {of_type}" @@ -95,7 +94,7 @@ def gen_schema(state: State) -> int: strict = state.conf.options.strict # Accessing this adds extra stuff to core, so we need to do it first - env_properties = _get_schema(state.envs["3.13"].conf, path="#/properties/env_run_base/properties") + env_properties = _get_schema(state.envs["py"].conf, path="#/properties/env_run_base/properties") properties = _get_schema(core, path="#/properties") diff --git a/tests/config/cli/conftest.py b/tests/config/cli/conftest.py index d5852bff8..69da08843 100644 --- a/tests/config/cli/conftest.py +++ b/tests/config/cli/conftest.py @@ -12,6 +12,7 @@ from tox.session.cmd.quickstart import quickstart from tox.session.cmd.run.parallel import run_parallel from tox.session.cmd.run.sequential import run_sequential +from tox.session.cmd.schema import gen_schema from tox.session.cmd.show_config import show_config if TYPE_CHECKING: @@ -23,6 +24,7 @@ def core_handlers() -> dict[str, Callable[[State], int]]: return { "config": show_config, "c": show_config, + "schema": gen_schema, "list": list_env, "l": list_env, "run": run_sequential, diff --git a/tests/session/cmd/test_schema.py b/tests/session/cmd/test_schema.py new file mode 100644 index 000000000..53e53b14d --- /dev/null +++ b/tests/session/cmd/test_schema.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + from tox.pytest import MonkeyPatch, ToxProjectCreator + + +def test_show_schema_empty_dir(tox_project: ToxProjectCreator, monkeypatch: MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + + project = tox_project({}) + result = project.run("-qq", "schema") + schema = json.loads(result.out) + assert "properties" in schema + assert "tox_root" in schema["properties"]