libs: Add envoy.docs.abstract

README.md

@@ -109,7 +109,7 @@ pypi: https://pypi.org/project/envoy.base.runner
 
 #### [envoy.base.utils](envoy.base.utils)
 
-version: 0.0.9
+version: 0.0.10.dev0
 
 pypi: https://pypi.org/project/envoy.base.utils
 
@@ -236,6 +236,23 @@ pypi: https://pypi.org/project/envoy.docker.utils
 ---
 
 
+#### [envoy.docs.abstract](envoy.docs.abstract)
+
+version: 0.0.1
+
+pypi: https://pypi.org/project/envoy.docs.abstract
+
+##### requirements:
+
+- [abstracts](https://pypi.org/project/abstracts)
+- [envoy.base.runner](https://pypi.org/project/envoy.base.runner)
+- [envoy.base.utils](https://pypi.org/project/envoy.base.utils) >=0.0.9
+- [jinja2](https://pypi.org/project/jinja2)
+- [protobuf](https://pypi.org/project/protobuf)
+
+---
+
+
 #### [envoy.docs.sphinx_runner](envoy.docs.sphinx_runner)
 
 version: 0.0.4.dev0

deps/requirements.txt

@@ -12,7 +12,7 @@ docutils~=0.16.0
 envoy.abstract.command>=0.0.3
 envoy.base.checker>=0.0.2
 envoy.base.runner>=0.0.4
-envoy.base.utils>=0.0.8
+envoy.base.utils>=0.0.9
 envoy.code_format.python_check>=0.0.4
 envoy.dependency.pip_check>=0.0.4
 envoy.distribution.distrotest>=0.0.3
@@ -27,6 +27,7 @@ envoy.gpg.sign>=0.0.3
 flake8
 frozendict
 gidgethub
+protobuf
 jinja2
 mypy
 mypy-abstracts
@@ -44,6 +45,7 @@ sphinxcontrib-httpdomain
 sphinxcontrib-serializinghtml
 sphinxext-rediraffe
 trycast
+types-protobuf
 types-setuptools
 verboselogs
 wheel-inspect

envoy.base.utils/VERSION

@@ -1 +1 @@
-0.0.9
+0.0.10-dev

envoy.docs.abstract/BUILD

@@ -0,0 +1,2 @@
+
+pytooling_package("envoy.docs.abstract")

envoy.docs.abstract/README.rst

@@ -0,0 +1,5 @@
+
+envoy.docs.abstract
+===================
+
+Abstract RST classes and utils used in Envoy proxy's CI

envoy.docs.abstract/VERSION

@@ -0,0 +1 @@
+0.0.1

envoy.docs.abstract/envoy/docs/abstract/BUILD

@@ -0,0 +1,12 @@
+
+pytooling_library(
+    "envoy.docs.abstract",
+    dependencies=[
+        "//deps:abstracts",
+        "//deps:envoy.base.utils",
+        "//deps:envoy.base.runner",
+        "//deps:frozendict",
+        "//deps:protobuf",
+        "//deps:jinja2",
+    ],
+)

envoy.docs.abstract/envoy/docs/abstract/__init__.py

@@ -0,0 +1,32 @@
+
+from .api import (
+    AAPIDocsBuilder,
+    APIFilesGenerator,
+    EmptyExtensionsDict,
+    ExtensionDetailsDict)
+from .builder import ADocsBuilder
+from .deps import ADependenciesDocsBuilder, RepositoryLocationsDict
+from .exceptions import RSTFormatterError
+from .extensions import (
+    AExtensionsDocsBuilder, ExtensionsMetadataDict,
+    ExtensionSecurityPosturesDict)
+from .formatter import ARSTFormatter, AProtobufRSTFormatter
+from .runner import ADocsBuildingRunner, BuildersDict
+
+
+__all__ = (
+    "AAPIDocsBuilder",
+    "ADependenciesDocsBuilder",
+    "AExtensionsDocsBuilder",
+    "ADocsBuilder",
+    "ADocsBuildingRunner",
+    "APIFilesGenerator",
+    "AProtobufRSTFormatter",
+    "ARSTFormatter",
+    "BuildersDict",
+    "EmptyExtensionsDict",
+    "ExtensionDetailsDict",
+    "ExtensionSecurityPosturesDict",
+    "ExtensionsMetadataDict",
+    "RepositoryLocationsDict",
+    "RSTFormatterError")

envoy.docs.abstract/envoy/docs/abstract/api.py

@@ -0,0 +1,117 @@
+
+import abc
+import pathlib
+import string
+import tarfile
+from functools import cached_property
+from typing import Dict, Generator, Optional, Tuple, Union
+
+import abstracts
+
+from .builder import ADocsBuilder
+from .formatter import ARSTFormatter
+
+
+APIFilesGenerator = Generator[Tuple[str, bytes], None, None]
+EmptyExtensionsDict = Dict[pathlib.Path, Union[str, bytes]]
+ExtensionDetailsDict = Dict[str, str]
+
+EMPTY_EXTENSION_DOCS_TEMPLATE = string.Template(
+    """$header
+
+$description
+
+$reflink
+
+This extension does not have a structured configuration, `google.protobuf.Empty
+<https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#empty>`_
+should be used instead.
+
+$extension
+""")
+
+
+class AAPIDocsBuilder(ADocsBuilder, metaclass=abstracts.Abstraction):
+
+    @cached_property
+    def api_extensions_root(self) -> pathlib.PurePosixPath:
+        return self.api_root.joinpath("config")
+
+    @property
+    def api_files(self) -> APIFilesGenerator:
+        with tarfile.open(self._api_files) as tar:
+            for member in tar.getmembers():
+                if member.isdir():
+                    continue
+                path = self.normalize_proto_path(member.path)
+                if path:
+                    content = tar.extractfile(member)
+                    if content:
+                        yield path, content.read()
+
+    @cached_property
+    def api_root(self) -> pathlib.PurePosixPath:
+        return pathlib.PurePosixPath("api-v3")
+
+    @property
+    @abc.abstractmethod
+    def empty_extensions(self) -> EmptyExtensionsDict:
+        raise NotImplementedError
+
+    @property
+    def empty_extension_template(self) -> string.Template:
+        return EMPTY_EXTENSION_DOCS_TEMPLATE
+
+    @property
+    @abc.abstractmethod
+    def rst_formatter(self) -> ARSTFormatter:
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def v3_proto_rst(self) -> Tuple[str, ...]:
+        raise NotImplementedError
+
+    async def build(self) -> None:
+        for path, content in self.api_files:
+            self.out(
+                self.api_root.joinpath(path),
+                content)
+        for empty_path, empty_content in self.empty_extensions.items():
+            self.out(
+                self.api_extensions_root.joinpath(empty_path),
+                empty_content)
+
+    def canonical(self, path: str) -> str:
+        if path.startswith("contrib/"):
+            path = path[8:]
+        if path.startswith("envoy/"):
+            path = path[6:]
+        return path
+
+    def format_ref(self, ref):
+        return self.rst_formatter.internal_link(
+            "configuration overview", ref)
+
+    def get_reflink(self, title: str, ref: Optional[str]) -> str:
+        return (
+            f"{title} {self.format_ref(ref)} ."
+            if ref
+            else "")
+
+    def normalize_proto_path(self, path) -> Optional[str]:
+        if "/pkg/" not in path:
+            return None
+        path = path.split('/pkg/')[1]
+        if path in self.v3_proto_rst:
+            return self.canonical(path)
+
+    def render_empty_extension(
+            self,
+            extension: str,
+            details: ExtensionDetailsDict) -> str:
+        return self.empty_extension_template.substitute(
+            header=self.rst_formatter.header(details['title'], "="),
+            description=details.get('description', ''),
+            reflink=self.get_reflink(details["title"], details.get("ref")),
+            extension=self.rst_formatter.extension(extension))

envoy.docs.abstract/envoy/docs/abstract/builder.py

@@ -0,0 +1,9 @@
+
+import abstracts
+
+
+class ADocsBuilder(metaclass=abstracts.Abstraction):
+
+    def __init__(self, out, api_files) -> None:
+        self.out = out
+        self._api_files = api_files