diff --git a/README.md b/README.md index 85bf943a0..9d5cb7fd2 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ pypi: https://pypi.org/project/envoy.base.runner #### [envoy.base.utils](envoy.base.utils) -version: 0.0.9 +version: 0.0.10.dev0 pypi: https://pypi.org/project/envoy.base.utils @@ -236,6 +236,23 @@ pypi: https://pypi.org/project/envoy.docker.utils --- +#### [envoy.docs.abstract](envoy.docs.abstract) + +version: 0.0.1 + +pypi: https://pypi.org/project/envoy.docs.abstract + +##### requirements: + +- [abstracts](https://pypi.org/project/abstracts) +- [envoy.base.runner](https://pypi.org/project/envoy.base.runner) +- [envoy.base.utils](https://pypi.org/project/envoy.base.utils) >=0.0.9 +- [jinja2](https://pypi.org/project/jinja2) +- [protobuf](https://pypi.org/project/protobuf) + +--- + + #### [envoy.docs.sphinx_runner](envoy.docs.sphinx_runner) version: 0.0.4.dev0 diff --git a/deps/requirements.txt b/deps/requirements.txt index 2aea9cb27..df146533e 100644 --- a/deps/requirements.txt +++ b/deps/requirements.txt @@ -12,7 +12,7 @@ docutils~=0.16.0 envoy.abstract.command>=0.0.3 envoy.base.checker>=0.0.2 envoy.base.runner>=0.0.4 -envoy.base.utils>=0.0.8 +envoy.base.utils>=0.0.9 envoy.code_format.python_check>=0.0.4 envoy.dependency.pip_check>=0.0.4 envoy.distribution.distrotest>=0.0.3 @@ -27,6 +27,7 @@ envoy.gpg.sign>=0.0.3 flake8 frozendict gidgethub +protobuf jinja2 mypy mypy-abstracts @@ -44,6 +45,7 @@ sphinxcontrib-httpdomain sphinxcontrib-serializinghtml sphinxext-rediraffe trycast +types-protobuf types-setuptools verboselogs wheel-inspect diff --git a/envoy.base.utils/VERSION b/envoy.base.utils/VERSION index c5d54ec32..4b9e5b025 100644 --- a/envoy.base.utils/VERSION +++ b/envoy.base.utils/VERSION @@ -1 +1 @@ -0.0.9 +0.0.10-dev diff --git a/envoy.docs.abstract/BUILD b/envoy.docs.abstract/BUILD new file mode 100644 index 000000000..2b329ce4e --- /dev/null +++ b/envoy.docs.abstract/BUILD @@ -0,0 +1,2 @@ + +pytooling_package("envoy.docs.abstract") diff --git a/envoy.docs.abstract/README.rst b/envoy.docs.abstract/README.rst new file mode 100644 index 000000000..43b5893d0 --- /dev/null +++ b/envoy.docs.abstract/README.rst @@ -0,0 +1,5 @@ + +envoy.docs.abstract +=================== + +Abstract RST classes and utils used in Envoy proxy's CI diff --git a/envoy.docs.abstract/VERSION b/envoy.docs.abstract/VERSION new file mode 100644 index 000000000..8acdd82b7 --- /dev/null +++ b/envoy.docs.abstract/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/envoy.docs.abstract/envoy/docs/abstract/BUILD b/envoy.docs.abstract/envoy/docs/abstract/BUILD new file mode 100644 index 000000000..2264a9761 --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/BUILD @@ -0,0 +1,12 @@ + +pytooling_library( + "envoy.docs.abstract", + dependencies=[ + "//deps:abstracts", + "//deps:envoy.base.utils", + "//deps:envoy.base.runner", + "//deps:frozendict", + "//deps:protobuf", + "//deps:jinja2", + ], +) diff --git a/envoy.docs.abstract/envoy/docs/abstract/__init__.py b/envoy.docs.abstract/envoy/docs/abstract/__init__.py new file mode 100644 index 000000000..27f9160b7 --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/__init__.py @@ -0,0 +1,32 @@ + +from .api import ( + AAPIDocsBuilder, + APIFilesGenerator, + EmptyExtensionsDict, + ExtensionDetailsDict) +from .builder import ADocsBuilder +from .deps import ADependenciesDocsBuilder, RepositoryLocationsDict +from .exceptions import RSTFormatterError +from .extensions import ( + AExtensionsDocsBuilder, ExtensionsMetadataDict, + ExtensionSecurityPosturesDict) +from .formatter import ARSTFormatter, AProtobufRSTFormatter +from .runner import ADocsBuildingRunner, BuildersDict + + +__all__ = ( + "AAPIDocsBuilder", + "ADependenciesDocsBuilder", + "AExtensionsDocsBuilder", + "ADocsBuilder", + "ADocsBuildingRunner", + "APIFilesGenerator", + "AProtobufRSTFormatter", + "ARSTFormatter", + "BuildersDict", + "EmptyExtensionsDict", + "ExtensionDetailsDict", + "ExtensionSecurityPosturesDict", + "ExtensionsMetadataDict", + "RepositoryLocationsDict", + "RSTFormatterError") diff --git a/envoy.docs.abstract/envoy/docs/abstract/api.py b/envoy.docs.abstract/envoy/docs/abstract/api.py new file mode 100644 index 000000000..4b199dc01 --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/api.py @@ -0,0 +1,117 @@ + +import abc +import pathlib +import string +import tarfile +from functools import cached_property +from typing import Dict, Generator, Optional, Tuple, Union + +import abstracts + +from .builder import ADocsBuilder +from .formatter import ARSTFormatter + + +APIFilesGenerator = Generator[Tuple[str, bytes], None, None] +EmptyExtensionsDict = Dict[pathlib.Path, Union[str, bytes]] +ExtensionDetailsDict = Dict[str, str] + +EMPTY_EXTENSION_DOCS_TEMPLATE = string.Template( + """$header + +$description + +$reflink + +This extension does not have a structured configuration, `google.protobuf.Empty +`_ +should be used instead. + +$extension +""") + + +class AAPIDocsBuilder(ADocsBuilder, metaclass=abstracts.Abstraction): + + @cached_property + def api_extensions_root(self) -> pathlib.PurePosixPath: + return self.api_root.joinpath("config") + + @property + def api_files(self) -> APIFilesGenerator: + with tarfile.open(self._api_files) as tar: + for member in tar.getmembers(): + if member.isdir(): + continue + path = self.normalize_proto_path(member.path) + if path: + content = tar.extractfile(member) + if content: + yield path, content.read() + + @cached_property + def api_root(self) -> pathlib.PurePosixPath: + return pathlib.PurePosixPath("api-v3") + + @property + @abc.abstractmethod + def empty_extensions(self) -> EmptyExtensionsDict: + raise NotImplementedError + + @property + def empty_extension_template(self) -> string.Template: + return EMPTY_EXTENSION_DOCS_TEMPLATE + + @property + @abc.abstractmethod + def rst_formatter(self) -> ARSTFormatter: + raise NotImplementedError + + @property + @abc.abstractmethod + def v3_proto_rst(self) -> Tuple[str, ...]: + raise NotImplementedError + + async def build(self) -> None: + for path, content in self.api_files: + self.out( + self.api_root.joinpath(path), + content) + for empty_path, empty_content in self.empty_extensions.items(): + self.out( + self.api_extensions_root.joinpath(empty_path), + empty_content) + + def canonical(self, path: str) -> str: + if path.startswith("contrib/"): + path = path[8:] + if path.startswith("envoy/"): + path = path[6:] + return path + + def format_ref(self, ref): + return self.rst_formatter.internal_link( + "configuration overview", ref) + + def get_reflink(self, title: str, ref: Optional[str]) -> str: + return ( + f"{title} {self.format_ref(ref)} ." + if ref + else "") + + def normalize_proto_path(self, path) -> Optional[str]: + if "/pkg/" not in path: + return None + path = path.split('/pkg/')[1] + if path in self.v3_proto_rst: + return self.canonical(path) + + def render_empty_extension( + self, + extension: str, + details: ExtensionDetailsDict) -> str: + return self.empty_extension_template.substitute( + header=self.rst_formatter.header(details['title'], "="), + description=details.get('description', ''), + reflink=self.get_reflink(details["title"], details.get("ref")), + extension=self.rst_formatter.extension(extension)) diff --git a/envoy.docs.abstract/envoy/docs/abstract/builder.py b/envoy.docs.abstract/envoy/docs/abstract/builder.py new file mode 100644 index 000000000..82d42789e --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/builder.py @@ -0,0 +1,9 @@ + +import abstracts + + +class ADocsBuilder(metaclass=abstracts.Abstraction): + + def __init__(self, out, api_files) -> None: + self.out = out + self._api_files = api_files diff --git a/envoy.docs.abstract/envoy/docs/abstract/deps.py b/envoy.docs.abstract/envoy/docs/abstract/deps.py new file mode 100644 index 000000000..4d47a939d --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/deps.py @@ -0,0 +1,177 @@ + +import abc +import pathlib +import urllib.parse +from collections import defaultdict, namedtuple +from typing import Any, Dict + +import abstracts + +from .builder import ADocsBuilder +from .formatter import ARSTFormatter + + +RepositoryLocationsDict = Dict[str, Any] + +CSV_TABLE_TEMPLATE = """.. csv-table:: + :header: {headers} + :widths: {widths} + + {csv_rows} + +""" + +NIST_CPE_URL_TEMPLATE = ( + "https://nvd.nist.gov/vuln/search/results?form_type=Advanced&" + "results_type=overview&query={encoded_cpe}&search_type=all") + + +# Obtain GitHub project URL from a list of URLs. +def get_github_project_url(urls): + for url in urls: + if not url.startswith('https://github.com/'): + continue + components = url.split('/') + return f'https://github.com/{components[3]}/{components[4]}' + return None + + +# Information releated to a GitHub release version. +GitHubRelease = namedtuple( + 'GitHubRelease', + ['organization', 'project', 'version', 'tagged']) + + +# Search through a list of URLs and determine if any contain a GitHub URL. If +# so, use heuristics to extract the release version and repo details, return +# this, otherwise return None. +def get_github_release_from_urls(urls): + for url in urls: + if not url.startswith('https://github.com/'): + continue + components = url.split('/') + if components[5] == 'archive': + # Only support .tar.gz, .zip today. Figure out the release tag from + # this filename. + if components[6].endswith('.tar.gz'): + github_version = components[6][:-len('.tar.gz')] + else: + assert (components[6].endswith('.zip')) + github_version = components[6][:-len('.zip')] + else: + # Release tag is a path component. + assert (components[5] == 'releases') + github_version = components[7] + # If it's not a GH hash, it's a tagged release. + tagged_release = len(github_version) != 40 + return GitHubRelease( + organization=components[3], + project=components[4], + version=github_version, + tagged=tagged_release) + return None + + +class ADependenciesDocsBuilder(ADocsBuilder, metaclass=abstracts.Abstraction): + + @property + def csv_table_template(self): + return CSV_TABLE_TEMPLATE + + @property + def extension_dependencies(self): + # Generate per-use category RST with CSV tables. + for category, exts in self.use_categories.items(): + content = '' + output_path = self.security_rst_root.joinpath( + f'external_dep_{category}.rst') + for ext_name, deps in sorted(exts.items()): + if ext_name != 'core': + content += self.rst_formatter.header(ext_name) + content += self.csv_table( + ['Name', 'Version', 'Release date', 'CPE'], [2, 1, 1, 2], + [[dep.name, dep.version, dep.release_date, dep.cpe] + for dep + in sorted(deps, key=lambda d: d.sort_name)]) + yield output_path, content + + @property + def nist_cpe_url_template(self): + return NIST_CPE_URL_TEMPLATE + + @property + @abc.abstractmethod + def repository_locations(self) -> RepositoryLocationsDict: + raise NotImplementedError + + @property + @abc.abstractmethod + def rst_formatter(self) -> ARSTFormatter: + raise NotImplementedError + + @property + def security_rst_root(self) -> pathlib.PurePosixPath: + return pathlib.PurePosixPath("intro/arch_overview/security") + + @property + def use_categories(self): + Dep = namedtuple( + 'Dep', + ['name', 'sort_name', 'version', 'cpe', 'release_date']) + use_categories = defaultdict(lambda: defaultdict(list)) + # Bin rendered dependencies into per-use category lists. + for k, v in self.repository_locations.items(): + cpe = v.get('cpe', '') + if cpe == 'N/A': + cpe = '' + if cpe: + cpe = self.rst_formatter.external_link( + cpe, self.nist_cpe_url(cpe)) + project_name = v['project_name'] + project_url = v['project_url'] + name = self.rst_formatter.external_link(project_name, project_url) + version = self.rst_formatter.external_link( + self.rst_formatter.version(v['version']), + self.get_version_url(v)) + release_date = v['release_date'] + dep = Dep(name, project_name.lower(), version, cpe, release_date) + for category in v['use_category']: + for ext in v.get('extensions', ['core']): + use_categories[category][ext].append(dep) + return use_categories + + async def build(self): + for output_path, content in self.extension_dependencies: + self.out(output_path, content) + + # Render a CSV table given a list of table headers, widths and list of rows + # (each a list of strings). + def csv_table(self, headers, widths, rows): + return self.csv_table_template.format( + headers=', '.join(headers), + csv_rows='\n '.join(', '.join(row) for row in rows), + widths=', '.join(str(w) for w in widths)) + + # Determine the version link URL. If it's GitHub, use some heuristics to + # figure out a release tag link, otherwise point to the GitHub tree at the + # respective SHA. Otherwise, return the tarball download. + def get_version_url(self, metadata): + # Figure out if it's a GitHub repo. + github_release = get_github_release_from_urls(metadata['urls']) + # If not, direct download link for tarball + if not github_release: + return metadata['urls'][0] + github_repo = ( + "https://github.com/" + f"{github_release.organization}/{github_release.project}") + if github_release.tagged: + # The GitHub version should look like the metadata version, + # but might have something like a "v" prefix. + return f'{github_repo}/releases/tag/{github_release.version}' + assert (metadata['version'] == github_release.version) + return f'{github_repo}/tree/{github_release.version}' + + # NIST CPE database search URL for a given CPE. + def nist_cpe_url(self, cpe): + return self.nist_cpe_url_template.format( + encoded_cpe=urllib.parse.quote(cpe)) diff --git a/envoy.docs.abstract/envoy/docs/abstract/exceptions.py b/envoy.docs.abstract/envoy/docs/abstract/exceptions.py new file mode 100644 index 000000000..e0c80b1aa --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/exceptions.py @@ -0,0 +1,4 @@ + + +class RSTFormatterError(Exception): + pass diff --git a/envoy.docs.abstract/envoy/docs/abstract/extensions.py b/envoy.docs.abstract/envoy/docs/abstract/extensions.py new file mode 100644 index 000000000..204785e6b --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/extensions.py @@ -0,0 +1,51 @@ + +import abc +import pathlib +from functools import cached_property +from typing import Any, List, Dict + +import abstracts + +from .builder import ADocsBuilder +from .formatter import ARSTFormatter + + +ExtensionsMetadataDict = Dict[str, Dict[str, Any]] +ExtensionSecurityPosturesDict = Dict[str, List[str]] + + +class AExtensionsDocsBuilder(ADocsBuilder, metaclass=abstracts.Abstraction): + + @property + @abc.abstractmethod + def extensions_metadata(self) -> ExtensionsMetadataDict: + raise NotImplementedError + + @property + @abc.abstractmethod + def rst_formatter(self) -> ARSTFormatter: + raise NotImplementedError + + @property + @abc.abstractmethod + def security_postures(self) -> ExtensionSecurityPosturesDict: + raise NotImplementedError + + @cached_property + def security_rst_root(self) -> pathlib.PurePosixPath: + return pathlib.PurePosixPath("intro/arch_overview/security") + + async def build(self) -> None: + for sp, extensions in self.security_postures.items(): + self.out( + self.security_rst_root.joinpath(f'secpos_{sp}.rst'), + self.render(extensions)) + + def render(self, extensions: List[str]) -> str: + return "\n".join( + self.rst_formatter.extension_list_item( + extension, + self.extensions_metadata[extension]) + for extension + in sorted(extensions) + if self.extensions_metadata[extension].get("status") != "wip") diff --git a/envoy.docs.abstract/envoy/docs/abstract/formatter.py b/envoy.docs.abstract/envoy/docs/abstract/formatter.py new file mode 100644 index 000000000..2081f0039 --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/formatter.py @@ -0,0 +1,755 @@ +import abc +from collections import defaultdict +import functools +import sys +from typing import Callable, Dict, Iterable, List, Tuple + +import yaml + +from frozendict import frozendict + +from jinja2 import Template + +from google.protobuf import descriptor_pb2 + +import abstracts + +from .exceptions import RSTFormatterError + + +CONTRIB_NOTE = """ + +.. note:: + This extension is only available in :ref:`contrib ` images. + +""" + +# Template for formating an extension category. +EXTENSION_CATEGORY_TEMPLATE = Template( + """ +.. _extension_category_{{category}}: + +.. tip:: +{% if extensions %} + This extension category has the following known extensions: + +{% for ext in extensions %} + - :ref:`{{ext}} ` +{% endfor %} + +{% endif %} +{% if contrib_extensions %} + The following extensions are available in :ref:`contrib ` + images only: + +{% for ext in contrib_extensions %} + - :ref:`{{ext}} ` +{% endfor %} +{% endif %} + +""") + +# Template for formating extension descriptions. +EXTENSION_TEMPLATE = Template( + """ +.. _extension_{{extension}}: + +This extension may be referenced by the qualified name ``{{extension}}`` +{{contrib}} +.. note:: + {{status}} + + {{security_posture}} + +.. tip:: + This extension extends and can be used with the following extension + {% if categories|length > 1 %}categories{% else %}category{% endif %}: + +{% for cat in categories %} + - :ref:`{{cat}} ` +{% endfor %} + +""") + +V2_LINK_TEMPLATE = Template( + """ +This documentation is for the Envoy v3 API. + +As of Envoy v1.18 the v2 API has been removed and is no longer supported. + +If you are upgrading from v2 API config you may wish to view the v2 API +documentation: + + :ref:`{{v2_text}} <{{v2_url}}>` + +""") + +# Namespace prefix for WKTs. +WKT_NAMESPACE_PREFIX = '.google.protobuf.' + +# Namespace prefix for RPCs. +RPC_NAMESPACE_PREFIX = '.google.rpc.' + +# Namespace prefix for Envoy core APIs. +ENVOY_API_NAMESPACE_PREFIX = '.envoy.api.v2.' + +# Namespace prefix for Envoy top-level APIs. +ENVOY_PREFIX = '.envoy.' + +# http://www.fileformat.info/info/unicode/char/2063/index.htm +UNICODE_INVISIBLE_SEPARATOR = u'\u2063' + +PROTOBUF_SCALAR_URL = ( + 'https://developers.google.com/protocol-buffers/docs/proto#scalar') +GOOGLE_RPC_URL_TPL = ( + "https://cloud.google.com/natural-language/docs/reference/rpc/" + "google.rpc#{rpc}") +PROTOBUF_URL_TPL = ( + "https://developers.google.com/protocol-buffers/docs/reference/" + "google.protobuf#{wkt}") +FIELD_TYPE_NAMES = frozendict({ + descriptor_pb2.FieldDescriptorProto.TYPE_DOUBLE: 'double', + descriptor_pb2.FieldDescriptorProto.TYPE_FLOAT: 'float', + descriptor_pb2.FieldDescriptorProto.TYPE_INT32: 'int32', + descriptor_pb2.FieldDescriptorProto.TYPE_SFIXED32: 'int32', + descriptor_pb2.FieldDescriptorProto.TYPE_SINT32: 'int32', + descriptor_pb2.FieldDescriptorProto.TYPE_FIXED32: 'uint32', + descriptor_pb2.FieldDescriptorProto.TYPE_UINT32: 'uint32', + descriptor_pb2.FieldDescriptorProto.TYPE_INT64: 'int64', + descriptor_pb2.FieldDescriptorProto.TYPE_SFIXED64: 'int64', + descriptor_pb2.FieldDescriptorProto.TYPE_SINT64: 'int64', + descriptor_pb2.FieldDescriptorProto.TYPE_FIXED64: 'uint64', + descriptor_pb2.FieldDescriptorProto.TYPE_UINT64: 'uint64', + descriptor_pb2.FieldDescriptorProto.TYPE_BOOL: 'bool', + descriptor_pb2.FieldDescriptorProto.TYPE_STRING: 'string', + descriptor_pb2.FieldDescriptorProto.TYPE_BYTES: 'bytes'}) +EXTRA_FIELD_TYPE_NAMES = frozendict({ + descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL: '', + descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED: '**repeated** '}) + + +class ARSTFormatter(metaclass=abstracts.Abstraction): + + @property + @abc.abstractmethod + def contrib_extensions_categories(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def contrib_note(self) -> str: + return CONTRIB_NOTE + + @property + @abc.abstractmethod + def envoy_last_v2_version(self) -> str: + raise NotImplementedError + + @property + @abc.abstractmethod + def extension_category_template(self) -> Template: + return EXTENSION_CATEGORY_TEMPLATE + + @property + @abc.abstractmethod + def extension_template(self) -> Template: + return EXTENSION_TEMPLATE + + @property + @abc.abstractmethod + def extensions_categories(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def extensions_metadata(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def extension_security_postures(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def extension_status_types(self): + raise NotImplementedError + + @property + def invisible_separator(self): + return UNICODE_INVISIBLE_SEPARATOR + + @property + @abc.abstractmethod + def pb(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def validate_fragment(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def v2_link_template(self) -> Template: + return V2_LINK_TEMPLATE + + @property + @abc.abstractmethod + def v2_mapping(self): + raise NotImplementedError + + def anchor(self, label) -> str: + """Format a label as an Envoy API RST anchor.""" + return f".. _{label}:\n\n" + + def extension_category(self, category: str) -> str: + """Format extension metadata as RST.""" + extensions, contrib_extensions = self._get_extensions(category) + if not (extensions or contrib_extensions): + raise RSTFormatterError( + "\n\nUnable to find extension category: " + f"{category}\n\n") + return self.extension_category_template.render( + category=category, + extensions=extensions, + contrib_extensions=contrib_extensions) + + def extension(self, extension: str) -> str: + """Format extension metadata as RST.""" + try: + extension_metadata = self.extensions_metadata.get(extension, None) + contrib = ( + self.contrib_note + if extension_metadata and extension_metadata.get("contrib") + else "") + status = self.extension_status_types.get( + extension_metadata.get('status'), '') + security_posture = self.extension_security_postures[ + extension_metadata['security_posture']] + categories = extension_metadata["categories"] + except KeyError: + sys.stderr.write( + f"\n\nDid you forget to add '{extension}' to " + "extensions_build_config.bzl, extensions_metadata.yaml, " + "contrib_build_config.bzl, or " + "contrib/extensions_metadata.yaml?\n\n") + # Raising the error buries the above message in tracebacks. + exit(1) + + return self.extension_template.render( + extension=extension, + contrib=contrib, + status=status, + security_posture=security_posture, + categories=categories) + + def extension_list_item(self, extension: str, metadata: Dict) -> str: + item = ( + f"* {extension}" + if metadata.get("undocumented") + else f"* :ref:`{extension} `") + if metadata.get("status") == "alpha": + item += " (alpha)" + if metadata.get("contrib"): + item += " (:ref:`contrib builds ` only)" + return item + + def external_link(self, text: str, url: str, suffix: str = "__") -> str: + return f"`{text} <{url}>`{suffix}" + + def header(self, title: str, underline: str = "~") -> str: + return f'\n{title}\n{underline * len(title)}\n\n' + + def indent(self, spaces: int, line: str) -> str: + """Indent a string.""" + return f"{' ' * spaces}{line}" + + def indent_lines(self, spaces: int, lines: Iterable) -> map: + """Indent a list of strings.""" + return map(functools.partial(self.indent, spaces), lines) + + def internal_link(self, text: str, ref: str) -> str: + return f":ref:`{text} <{ref}>`" + + def map_lines(self, f: Callable, s: str) -> str: + """Apply a function across each line in a flat string.""" + return '\n'.join(f(line) for line in s.split('\n')) + + def strip_leading_space(self, lines) -> str: + """Remove leading space in flat comment strings.""" + return self.map_lines(lambda s: s[1:], lines) + + def v2_link(self, name: str) -> str: + if name not in self.v2_mapping: + return "" + v2_filepath = f"envoy_api_file_{self.v2_mapping[name]}" + return self.v2_link_template.render( + v2_url=f"v{self.envoy_last_v2_version}:{v2_filepath}", + v2_text=v2_filepath.split('/', 1)[1]) + + def version(self, version): + # Render version strings human readable. + # Heuristic, almost certainly a git SHA + if len(version) == 40: + # Abbreviate git SHA + return version[:7] + return version + + def _get_extensions(self, category: str) -> Tuple[List, List]: + return ( + sorted(self.extensions_categories.get(category, [])), + sorted(self.contrib_extensions_categories.get(category, []))) + + +class AProtobufRSTFormatter(metaclass=abstracts.Abstraction): + + def __init__(self, rst: ARSTFormatter): + self.rst = rst + + @property + @abc.abstractmethod + def annotations(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def api_namespace(self) -> str: + return ENVOY_API_NAMESPACE_PREFIX + + @property + @abc.abstractmethod + def extra_field_type_names(self): + return EXTRA_FIELD_TYPE_NAMES + + @property + @abc.abstractmethod + def field_type_names(self): + return FIELD_TYPE_NAMES + + @property + @abc.abstractmethod + def json_format(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def namespace(self) -> str: + return ENVOY_PREFIX + + @property + @abc.abstractmethod + def protodoc_manifest(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def security_pb2(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def status_pb2(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def validate_pb2(self): + raise NotImplementedError + + def comment_with_annotations(self, comment, type_name: str = '') -> str: + """Format a comment string with additional RST for annotations.""" + alpha_warning = '' + # if self.annotations.ALPHA_ANNOTATION in comment.annotations: + # experimental_warning = ( + # '.. warning::\n This API is alpha and is not covered ' + # 'by the :ref:`threat model `.\n\n' + # ) + extension = comment.annotations.get( + self.annotations.EXTENSION_ANNOTATION) + formatted_extension = ( + self.rst.extension(extension) + if extension + else "") + category_annotations = comment.annotations.get( + self.annotations.EXTENSION_CATEGORY_ANNOTATION, + "").split(",") + formatted_category = "".join( + self.rst.extension_category(category) + for category + in category_annotations + if category) + comment = self.annotations.without_annotations( + f"{self.rst.strip_leading_space(comment.raw)}\n") + return ( + f"{alpha_warning}{comment}{formatted_extension}" + f"{formatted_category}") + + def enum_as_dl(self, type_context, enum) -> str: + """Format a EnumDescriptorProto as RST definition list.""" + out = [] + for index, value in enumerate(enum.value): + ctx = type_context.extend_enum_value(index, value.name) + out.append( + self.enum_value_as_dl_item( + ctx.name, ctx.leading_comment, value)) + return "%s\n" % '\n'.join(out) + + def enum_value_as_dl_item( + self, + name: str, + comment: str, + enum_value) -> str: + """Format a EnumValueDescriptorProto as RST definition list item.""" + if self.hide_not_implemented(comment): + return '' + anchor = self.rst.anchor( + self.cross_ref_label( + self.normalize_field_type_name(f".{name}"), + "enum_value")) + default_comment = ( + "*(DEFAULT)* " + if enum_value.number == 0 + else "") + leading_comment = self.comment_with_annotations(comment) + comment = ( + f"{default_comment}{self.rst.invisible_separator}" + f"{leading_comment}") + lines = self.rst.map_lines( + functools.partial(self.rst.indent, 2), + comment) + return ( + f"{anchor}{enum_value.name}\n" + f"{lines}") + + def field_as_dl_item( + self, + outer_type_context, + type_context, + field: descriptor_pb2.FieldDescriptorProto) -> str: + """Format a FieldDescriptorProto as RST definition list item.""" + leading_comment = self.comment_with_annotations( + type_context.leading_comment) + + if self.hide_not_implemented(type_context.leading_comment): + return '' + + field_annotations = [] + anchor = self.rst.anchor( + self.cross_ref_label( + self.normalize_field_type_name(f".{type_context.name}"), + "field")) + + validate_rule = self._get_extension(field, self.validate_pb2.rules) + if validate_rule and self._is_required(validate_rule): + field_annotations = ['*REQUIRED*'] + required, oneof_comment = self.oneof_comment( + outer_type_context, type_context, field) + if required is None: + return "" + if not required: + field_annotations = [] + + comment = '(%s) ' % ', '.join( + [self.extra_field_type_names[field.label] + + self.field_type(type_context, field)] + + field_annotations) + leading_comment + lines = self.rst.map_lines( + functools.partial(self.rst.indent, 2), + f"{comment}{oneof_comment}") + # If there is a udpa.annotations.security option, include it after + # the comment. + return ( + f"{anchor}{field.name}\n{lines}" + f"{self.security_extension(type_context.name, field)}") + + def field_type( + self, + type_context, + field: descriptor_pb2.FieldDescriptorProto) -> str: + """Format a FieldDescriptorProto type description.""" + if self._is_envoy_field(field): + return self._envoy_field_type(type_context, field) + elif field.type_name.startswith(WKT_NAMESPACE_PREFIX): + wkt = field.type_name[len(WKT_NAMESPACE_PREFIX):] + return self.rst.external_link( + wkt, PROTOBUF_URL_TPL.format(wkt=wkt.lower())) + elif field.type_name.startswith(RPC_NAMESPACE_PREFIX): + rpc = field.type_name[len(RPC_NAMESPACE_PREFIX):] + return self.rst.external_link( + rpc, GOOGLE_RPC_URL_TPL.format(rpc=rpc.lower())) + elif field.type_name: + return field.type_name + if field.type in self.field_type_names: + return self.rst.external_link( + self.field_type_names[field.type], + PROTOBUF_SCALAR_URL) + raise RSTFormatterError('Unknown field type ' + str(field.type)) + + def field_type_as_json( + self, + type_context, + field: descriptor_pb2.FieldDescriptorProto) -> str: + """Format FieldDescriptorProto.Type as a pseudo-JSON string.""" + if field.label == field.LABEL_REPEATED: + return '[]' + return_object = ( + (field.type == field.TYPE_MESSAGE) + or (self.type_name_from_fqn(field.type_name) + in type_context.map_typenames)) + return ( + '"{...}"' + if return_object + else '"..."') + + def cross_ref_label(self, name: str, type: str) -> str: + return f"envoy_v3_api_{type}_{name}" + + def header_from_file( + self, + style, + source_code_info, + proto_name, + v2_link) -> Tuple[str, str]: + """Format RST header based on special file level title.""" + anchor = self.rst.anchor( + self.cross_ref_label(proto_name, "file")) + stripped_comment = self.annotations.without_annotations( + self.rst.strip_leading_space( + "\n".join( + f"{c}\n" + for c + in source_code_info.file_level_comments))) + formatted_extension = '' + extension_annotation = source_code_info.file_level_annotations.get( + self.annotations.EXTENSION_ANNOTATION) + if extension_annotation: + formatted_extension = self.rst.extension( + extension_annotation) + doc_title_annotation = source_code_info.file_level_annotations.get( + self.annotations.DOC_TITLE_ANNOTATION) + if doc_title_annotation: + return ( + f"{anchor}" + f"{self.rst.header(doc_title_annotation, style)}" + f"{v2_link}\n\n{formatted_extension}", + stripped_comment) + return ( + f"{anchor}{self.rst.header(proto_name, style)}{v2_link}" + f"\n\n{formatted_extension}", + stripped_comment) + + def hide_not_implemented(self, comment) -> bool: + """Hide comments marked with [#not-implemented-hide:]""" + return bool( + self.annotations.NOT_IMPLEMENTED_HIDE_ANNOTATION + in comment.annotations) + + def oneof_comment( + self, + outer_type_context, + type_context, + field): + if not field.HasField('oneof_index'): + return True, "" + oneof_context = outer_type_context.extend_oneof( + field.oneof_index, + type_context.oneof_names[field.oneof_index]) + + if self.hide_not_implemented(oneof_context.leading_comment): + return None, "" + oneof_comment = self.comment_with_annotations( + oneof_context.leading_comment) + + # If the oneof only has one field and marked required, mark the + # field as required. + required = ( + len(type_context.oneof_fields[field.oneof_index]) == 1 + and type_context.oneof_required[field.oneof_index]) + + if len(type_context.oneof_fields[field.oneof_index]) > 1: + # Fields in oneof shouldn't be marked as required when we have + # oneof comment below it. + required = False + oneof_comment += self._oneof_comment( + outer_type_context, type_context, field) + return required, oneof_comment + + def message_as_json(self, type_context, msg) -> str: + """Format a message definition DescriptorProto as a pseudo-JSON + block.""" + lines = [] + for index, field in enumerate(msg.field): + field_type_context = type_context.extend_field(index, field.name) + if self.hide_not_implemented(field_type_context.leading_comment): + continue + lines.append( + f'"{field.name}": ' + f"{self.field_type_as_json(type_context, field)}") + if lines: + return ( + ".. code-block:: json\n\n {\n%s\n }\n\n" + % ",\n".join(self.rst.indent_lines(4, lines))) + return "" + + def message_as_dl(self, type_context, msg) -> str: + """Format a DescriptorProto as RST definition list.""" + type_context.oneof_fields = defaultdict(list) + type_context.oneof_required = defaultdict(bool) + type_context.oneof_names = defaultdict(list) + for index, field in enumerate(msg.field): + if field.HasField('oneof_index'): + leading_comment = type_context.extend_field( + index, field.name).leading_comment + if self.hide_not_implemented(leading_comment): + continue + type_context.oneof_fields[field.oneof_index].append( + (index, field.name)) + for index, oneof_decl in enumerate(msg.oneof_decl): + if oneof_decl.options.HasExtension(self.validate_pb2.required): + type_context.oneof_required[index] = ( + oneof_decl.options.Extensions[self.validate_pb2.required]) + type_context.oneof_names[index] = oneof_decl.name + return '\n'.join( + self.field_as_dl_item( + type_context, + type_context.extend_field(index, field.name), + field) + for index, field in enumerate(msg.field)) + '\n' + + def normalize_field_type_name(self, field_fqn: str) -> str: + """Normalize a fully qualified field type name, e.g. + + .envoy.foo.bar. + + Strips leading ENVOY_API_NAMESPACE_PREFIX and ENVOY_PREFIX. + """ + if field_fqn.startswith(self.api_namespace): + return field_fqn[len(self.api_namespace):] + if field_fqn.startswith(self.namespace): + return field_fqn[len(self.namespace):] + return field_fqn + + def security_extension( + self, + name: str, + field: descriptor_pb2.FieldDescriptorProto): + sec_extension = self._get_extension(field, self.security_pb2.security) + if not sec_extension: + return "" + manifest = self.protodoc_manifest.fields.get(name) + if not manifest: + raise RSTFormatterError( + f"Missing protodoc manifest YAML for {name}") + return self.security_options( + sec_extension, + field, + name, + manifest.edge_config) + + def security_options( + self, + security_option, + field: descriptor_pb2.FieldDescriptorProto, + name, + edge_config) -> str: + sections = [] + if security_option.configure_for_untrusted_downstream: + sections.append( + self.rst.indent( + 4, + ("This field should be configured in the presence of " + "untrusted *downstreams*."))) + if security_option.configure_for_untrusted_upstream: + sections.append( + self.rst.indent( + 4, + ("This field should be configured in the presence of " + "untrusted *upstreams*."))) + if edge_config.note: + sections.append(self.rst.indent(4, edge_config.note)) + + example_dict = self.json_format.MessageToDict(edge_config.example) + self.rst.validate_fragment(field.type_name[1:], example_dict) + field_name = name.split('.')[-1] + example = {field_name: example_dict} + sections.append( + "".join([ + self.rst.indent( + 4, + 'Example configuration for untrusted environments:\n\n'), + self.rst.indent( + 4, + '.. code-block:: yaml\n\n'), + '\n'.join( + self.rst.indent_lines( + 6, + yaml.dump(example).split('\n')))])) + joined_sections = '\n\n'.join(sections) + return f".. attention::\n{joined_sections}" + + def type_name_from_fqn(self, fqn: str) -> str: + return fqn[1:] + + def _envoy_field_type( + self, + type_context, + field: descriptor_pb2.FieldDescriptorProto) -> str: + normal_type_name = self.normalize_field_type_name(field.type_name) + if field.type == field.TYPE_MESSAGE: + type_name = self.type_name_from_fqn(field.type_name) + if type_name in (type_context.map_typenames or []): + return ( + 'map<%s, %s>' + % tuple( + map(functools.partial(self.field_type, type_context), + type_context.map_typenames[type_name]))) + return self.rst.internal_link( + normal_type_name, + self.cross_ref_label(normal_type_name, "msg")) + if field.type == field.TYPE_ENUM: + return self.rst.internal_link( + normal_type_name, + self.cross_ref_label(normal_type_name, "enum")) + + def _is_envoy_field( + self, + field: descriptor_pb2.FieldDescriptorProto) -> bool: + return bool( + field.type_name.startswith(self.api_namespace) + or field.type_name.startswith(self.namespace)) + + def _is_required(self, rule): + return ( + (rule.HasField('message') and rule.message.required) + or (rule.HasField('duration') and rule.duration.required) + or (rule.HasField('string') and rule.string.min_len > 0) + or (rule.HasField('string') and rule.string.min_bytes > 0) + or (rule.HasField('repeated') and rule.repeated.min_items > 0)) + + def _oneof_comment( + self, + outer_type_context, + type_context, + field: descriptor_pb2.FieldDescriptorProto) -> str: + oneof_template = ( + '\nPrecisely one of %s must be set.\n' + if type_context.oneof_required[field.oneof_index] + else '\nOnly one of %s may be set.\n') + return ( + oneof_template + % ', '.join( + self.rst.internal_link( + f, + self.cross_ref_label( + self.normalize_field_type_name( + f".{outer_type_context.extend_field(i, f).name}"), + "field")) + for i, f in type_context.oneof_fields[field.oneof_index])) + + def _get_extension( + self, + field: descriptor_pb2.FieldDescriptorProto, + extension): + if field.options.HasExtension(extension): + return field.options.Extensions[extension] diff --git a/envoy.docs.abstract/envoy/docs/abstract/py.typed b/envoy.docs.abstract/envoy/docs/abstract/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/envoy.docs.abstract/envoy/docs/abstract/runner.py b/envoy.docs.abstract/envoy/docs/abstract/runner.py new file mode 100644 index 000000000..87b41ee32 --- /dev/null +++ b/envoy.docs.abstract/envoy/docs/abstract/runner.py @@ -0,0 +1,66 @@ + +import abc +import argparse +import io +import pathlib +import tarfile +from functools import cached_property +from typing import Any, Dict, Tuple, Type, Union + +import abstracts + +from envoy.base import runner, utils + +from .builder import ADocsBuilder + + +BuildersDict = Dict[str, Any] + + +class ADocsBuildingRunner(runner.AsyncRunner, metaclass=abstracts.Abstraction): + _builders: Tuple[Tuple[str, Type[ADocsBuilder]], ...] = () + + @classmethod + def register_builder(cls, name: str, util: Type[ADocsBuilder]) -> None: + """Register a repo type.""" + cls._builders = ( + getattr(cls, "_builders") + + ((name, util),)) + + @property + def api_rst_files(self): + return self.args.api_files + + @property + @abc.abstractmethod + def builders(self) -> BuildersDict: + return { + k: v(self.write_tar, self.api_rst_files) + for k, v in self._builders} + + @cached_property + def tar(self): + return tarfile.open(self.args.out_path, "w") + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + super().add_arguments(parser) + parser.add_argument( + "api_files", + help="Tarball containing Protobuf API rst files") + parser.add_argument( + "--out_path", + help="Outfile to tar rst files into") + + @abc.abstractmethod + async def run(self): + for builder in self.builders.values(): + await builder.build() + self.tar.close() + + def write_tar( + self, + path: Union[str, pathlib.Path], + content: Union[str, bytes]) -> None: + tarinfo = tarfile.TarInfo(str(path)) + tarinfo.size = len(content) + self.tar.addfile(tarinfo, io.BytesIO(utils.to_bytes(content))) diff --git a/envoy.docs.abstract/setup.cfg b/envoy.docs.abstract/setup.cfg new file mode 100644 index 000000000..f08451f3e --- /dev/null +++ b/envoy.docs.abstract/setup.cfg @@ -0,0 +1,51 @@ +[metadata] +name = envoy.docs.abstract +version = file: VERSION +author = Ryan Northey +author_email = ryan@synca.io +maintainer = Ryan Northey +maintainer_email = ryan@synca.io +license = Apache Software License 2.0 +url = https://github.com/envoyproxy/pytooling/tree/main/envoy.docs.abstract +description = "Abstract RST classes and utils used in Envoy proxy's CI" +long_description = file: README.rst +classifiers = + Development Status :: 4 - Beta + Framework :: Pytest + Intended Audience :: Developers + Topic :: Software Development :: Testing + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: Implementation :: CPython + Operating System :: OS Independent + License :: OSI Approved :: Apache Software License + +[options] +python_requires = >=3.5 +py_modules = envoy.docs.abstract +packages = find_namespace: +install_requires = + abstracts + envoy.base.utils>=0.0.9 + envoy.base.runner + frozendict + jinja2 + protobuf + +[options.extras_require] +test = + pytest + pytest-asyncio + pytest-coverage + pytest-patches +lint = flake8 +types = + mypy + types-protobuf +publish = wheel + +[options.package_data] +* = py.typed diff --git a/envoy.docs.abstract/setup.py b/envoy.docs.abstract/setup.py new file mode 100644 index 000000000..1f6a64b9c --- /dev/null +++ b/envoy.docs.abstract/setup.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +from setuptools import setup # type:ignore + +setup() diff --git a/envoy.docs.abstract/tests/BUILD b/envoy.docs.abstract/tests/BUILD new file mode 100644 index 000000000..19922f0e6 --- /dev/null +++ b/envoy.docs.abstract/tests/BUILD @@ -0,0 +1,9 @@ + +pytooling_tests( + "envoy.docs.abstract", + dependencies=[ + "//deps:abstracts", + "//deps:jinja2", + "//deps:pytest-asyncio", + ], +) diff --git a/envoy.docs.abstract/tests/test_formatter.py b/envoy.docs.abstract/tests/test_formatter.py new file mode 100644 index 000000000..0c63fa202 --- /dev/null +++ b/envoy.docs.abstract/tests/test_formatter.py @@ -0,0 +1,180 @@ + +import pytest + +import abstracts + +from envoy.docs import abstract + + +@abstracts.implementer(abstract.AProtobufRSTFormatter) +class DummyProtobufRSTFormatter: + + @property + def annotations(self): + return super().annotations + + @property + def api_namespace(self): + return super().api_namespace + + @property + def extra_field_type_names(self): + return super().extra_field_type_names + + @property + def field_type_names(self): + return super().field_type_names + + @property + def json_format(self): + return super().json_format + + @property + def namespace(self): + return super().namespace + + @property + def protodoc_manifest(self): + return super().protodoc_manifest + + @property + def status_pb2(self): + return super().status_pb2 + + @property + def security_pb2(self): + return super().security_pb2 + + @property + def validate_pb2(self): + return super().validate_pb2 + + +@abstracts.implementer(abstract.ARSTFormatter) +class DummyRSTFormatter: + + @property + def contrib_note(self): + return super().contrib_note + + @property + def contrib_extensions_categories(self): + return super().contrib_extensions_categories + + @property + def validate_fragment(self): + return super().validate_fragment + + @property + def v2_mapping(self): + return super().v2_mapping + + @property + def v2_link_template(self): + return super().v2_link_template + + @property + def pb(self): + return super().pb + + @property + def extensions_categories(self): + return super().extensions_categories + + @property + def envoy_last_v2_version(self): + return super().envoy_last_v2_version + + @property + def extension_category_template(self): + return super().extension_category_template + + @property + def extensions_metadata(self): + return super().extensions_metadata + + @property + def extension_security_postures(self): + return super().extension_security_postures + + @property + def extension_status_categories(self): + return super().extension_status_categories + + @property + def extension_status_types(self): + return super().extension_status_types + + @property + def extension_template(self): + return super().extension_template + + +def test_utils_formatter_constructor(): + with pytest.raises(TypeError): + abstract.ARSTFormatter() + + format = DummyRSTFormatter() + + iface_props = ( + "contrib_extensions_categories", "envoy_last_v2_version", + "extensions_categories", "extensions_metadata", + "extension_security_postures", "extension_status_types", "pb", + "validate_fragment", "v2_mapping") + + for prop in iface_props: + with pytest.raises(NotImplementedError): + getattr(format, prop) + + +@pytest.mark.parametrize( + "constant", + ("contrib_note", "extension_template")) +def test_utils_formatter_abstract_methods(constant): + assert ( + getattr(DummyRSTFormatter(), constant) + == getattr(abstract.formatter, constant.upper())) + + +@pytest.mark.parametrize("text", ("ABC", "", "ABCDXYZ")) +@pytest.mark.parametrize("url", ("URL1", "URL2")) +@pytest.mark.parametrize("suffix", (None, "_", "__")) +def test_utils_formatter_external_link(text, url, suffix): + args = (text, url, suffix) if suffix else (text, url) + suffix = suffix or "__" + assert ( + DummyRSTFormatter().external_link(*args) + == f"`{text} <{url}>`{suffix}") + + +@pytest.mark.parametrize("text", ("ABC", "", "ABCDXYZ")) +@pytest.mark.parametrize("ref", ("REF1", "REF2")) +def test_utils_formatter_internal_link(text, ref): + assert ( + DummyRSTFormatter().internal_link(text, ref) + == f":ref:`{text} <{ref}>`") + + +@pytest.mark.parametrize("text", ("ABC", "", "ABCDXYZ")) +@pytest.mark.parametrize("underline", (None, "*", "-", "~")) +def test_utils_formatter_header(text, underline): + args = (text, underline) if underline else (text, ) + underline = underline or "~" + assert ( + DummyRSTFormatter().header(*args) + == f'\n{text}\n{underline * len(text)}\n\n') + + +def test_utils_pb_formatter_constructor(): + with pytest.raises(TypeError): + abstract.AProtobufRSTFormatter("FORMATTER") + + proto_format = DummyProtobufRSTFormatter("FORMATTER") + + iface_props = ( + "annotations", "json_format", "protodoc_manifest", + "security_pb2", "status_pb2", "validate_pb2") + + for prop in iface_props: + with pytest.raises(NotImplementedError): + getattr(proto_format, prop) diff --git a/envoy.docs.abstract/tests/test_runner.py b/envoy.docs.abstract/tests/test_runner.py new file mode 100644 index 000000000..9e1c03d1b --- /dev/null +++ b/envoy.docs.abstract/tests/test_runner.py @@ -0,0 +1,106 @@ + +from unittest.mock import AsyncMock, MagicMock, PropertyMock + +import pytest + +import abstracts + +from envoy.docs import abstract + + +@abstracts.implementer(abstract.ADocsBuildingRunner) +class DummyDocsBuildingRunner: + + @property + def builders(self): + return super().builders + + async def run(self): + return await super().run() + + +def test_runner_constructor(): + with pytest.raises(TypeError): + abstract.ADocsBuildingRunner() + + DummyDocsBuildingRunner() + + +def test_runner_cls_register_builder(): + assert abstract.ADocsBuildingRunner._builders == () + + class Builder1(object): + pass + + class Builder2(object): + pass + + abstract.ADocsBuildingRunner.register_builder("builder1", Builder1) + assert ( + abstract.ADocsBuildingRunner._builders + == (('builder1', Builder1),)) + + abstract.ADocsBuildingRunner.register_builder("builder2", Builder2) + assert ( + abstract.ADocsBuildingRunner._builders + == (('builder1', Builder1), + ('builder2', Builder2),)) + + +@pytest.mark.asyncio +async def test_runner_run(patches): + runner = DummyDocsBuildingRunner() + patched = patches( + ("ADocsBuildingRunner.builders", + dict(new_callable=PropertyMock)), + ("ADocsBuildingRunner.tar", + dict(new_callable=PropertyMock)), + prefix="envoy.docs.abstract.runner") + builders = [AsyncMock() for i in range(0, 5)] + + with patched as (m_builders, m_tar): + m_builders.return_value.values.return_value = builders + assert not await runner.run() + + assert ( + list(m_builders.return_value.values.call_args) + == [(), {}]) + for builder in builders: + assert ( + list(builder.build.call_args) + == [(), {}]) + assert ( + list(m_tar.return_value.close.call_args) + == [(), {}]) + + +def test_runner_write_tar(patches): + runner = DummyDocsBuildingRunner() + patched = patches( + "io", + "utils", + "tarfile", + ("ADocsBuildingRunner.tar", + dict(new_callable=PropertyMock)), + prefix="envoy.docs.abstract.runner") + path = MagicMock() + content = MagicMock() + content.__len__.return_value = 23 + + with patched as (m_io, m_utils, m_tarfile, m_tar): + assert not runner.write_tar(path, content) + + assert ( + list(m_tarfile.TarInfo.call_args) + == [(str(path), ), {}]) + assert m_tarfile.TarInfo.return_value.size == 23 + assert ( + list(m_io.BytesIO.call_args) + == [(m_utils.to_bytes.return_value, ), {}]) + assert ( + list(m_utils.to_bytes.call_args) + == [(content, ), {}]) + assert ( + list(m_tar.return_value.addfile.call_args) + == [(m_tarfile.TarInfo.return_value, + m_io.BytesIO.return_value), {}]) diff --git a/pants.toml b/pants.toml index 0c032d7ad..50ac5ad68 100644 --- a/pants.toml +++ b/pants.toml @@ -44,6 +44,7 @@ extra_requirements = [ "mypy-abstracts", "types-aiofiles", "types-frozendict", + "types-protobuf", "types-pyyaml"] args = [ "--explicit-package-bases",