diff --git a/docs/source/conf.py b/docs/source/conf.py index 89ef62aedc3..13a734746d0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -70,9 +70,14 @@ sys.path += [os.path.dirname(__file__) + "/../"] extensions.append('util.cmdref') -def setup(sphinx): - from util.RtlilLexer import RtlilLexer - sphinx.add_lexer("RTLIL", RtlilLexer) +# use autodocs +extensions.append('sphinx.ext.autodoc') +extensions.append('util.cellref') - from util.YoscryptLexer import YoscryptLexer - sphinx.add_lexer("yoscrypt", YoscryptLexer) \ No newline at end of file +from sphinx.application import Sphinx +def setup(app: Sphinx) -> None: + from util.RtlilLexer import RtlilLexer + app.add_lexer("RTLIL", RtlilLexer) + + from util.YoscryptLexer import YoscryptLexer + app.add_lexer("yoscrypt", YoscryptLexer) diff --git a/docs/util/cellref.py b/docs/util/cellref.py new file mode 100644 index 00000000000..4deda41dddb --- /dev/null +++ b/docs/util/cellref.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +from pathlib import Path +import re + +from typing import Any +from sphinx.application import Sphinx +from sphinx.ext import autodoc +from sphinx.ext.autodoc import Documenter +from sphinx.util import logging + +logger = logging.getLogger(__name__) + +# cell signature +cell_ext_sig_re = re.compile( + r'''^ (?:([\w._/]+):)? # explicit file name + ([\w$._]+?)? # module and/or class name(s) + (?:\.([\w_]+))? # optional: thing name + (::[\w_]+)? # attribute + \s* $ # and nothing more + ''', re.VERBOSE) + +class SimHelper: + name: str = "" + title: str = "" + ports: str = "" + source: str = "" + desc: list[str] + code: list[str] + group: str = "" + ver: str = "1" + + def __init__(self) -> None: + self.desc = [] + +def simcells_reparse(cell: SimHelper): + # cut manual signature + cell.desc = cell.desc[3:] + + # code-block truth table + new_desc = [] + indent = "" + for line in cell.desc: + if line.startswith("Truth table:"): + indent = " " + new_desc.pop() + new_desc.extend(["::", ""]) + new_desc.append(indent + line) + cell.desc = new_desc + + # set version + cell.ver = "2a" + +def load_cell_lib(file: Path): + simHelpers: dict[str, SimHelper] = {} + simHelper = SimHelper() + with open(file, "r") as f: + lines = f.readlines() + + for lineno, line in enumerate(lines, 1): + line = line.rstrip() + # special comments + if line.startswith("//-"): + simHelper.desc.append(line[4:] if len(line) > 4 else "") + elif line.startswith("//* "): + _, key, val = line.split(maxsplit=2) + setattr(simHelper, key, val) + + # code parsing + if line.startswith("module "): + clean_line = line[7:].replace("\\", "").replace(";", "") + simHelper.name, simHelper.ports = clean_line.split(maxsplit=1) + simHelper.code = [] + simHelper.source = f'{file.name}:{lineno}' + elif not line.startswith("endmodule"): + line = " " + line + try: + simHelper.code.append(line.replace("\t", " ")) + except AttributeError: + # no module definition, ignore line + pass + if line.startswith("endmodule"): + if simHelper.ver == "1" and file.name == "simcells.v": + # default simcells parsing + simcells_reparse(simHelper) + if not simHelper.desc: + # no help + simHelper.desc.append("No help message for this cell type found.\n") + elif simHelper.ver == "1" and file.name == "simlib.v" and simHelper.desc[1].startswith(' '): + simHelper.desc.pop(1) + simHelpers[simHelper.name] = simHelper + simHelper = SimHelper() + return simHelpers + +class YosysCellDocumenter(Documenter): + objtype = 'cell' + parsed_libs: dict[Path, dict[str, SimHelper]] = {} + object: SimHelper + + option_spec = { + 'source': autodoc.bool_option, + 'linenos': autodoc.bool_option, + } + + @classmethod + def can_document_member( + cls, + member: Any, + membername: str, + isattr: bool, + parent: Any + ) -> bool: + sourcename = str(member).split(":")[0] + if not sourcename.endswith(".v"): + return False + if membername == "__source": + return False + + def parse_name(self) -> bool: + try: + matched = cell_ext_sig_re.match(self.name) + path, modname, thing, attribute = matched.groups() + except AttributeError: + logger.warning(('invalid signature for auto%s (%r)') % (self.objtype, self.name), + type='cellref') + return False + + if not path: + return False + + self.modname = modname + self.objpath = [path] + self.attribute = attribute + self.fullname = ((self.modname) + (thing or '')) + + return True + + def import_object(self, raiseerror: bool = False) -> bool: + # find cell lib file + objpath = Path('/'.join(self.objpath)) + if not objpath.exists(): + objpath = Path('source') / 'generated' / objpath + + # load cell lib + try: + parsed_lib = self.parsed_libs[objpath] + except KeyError: + parsed_lib = load_cell_lib(objpath) + self.parsed_libs[objpath] = parsed_lib + + # get cell + try: + self.object = parsed_lib[self.modname] + except KeyError: + return False + + self.real_modname = f'{objpath}:{self.modname}' + return True + + def get_sourcename(self) -> str: + return self.objpath + + def format_name(self) -> str: + return self.object.name + + def format_signature(self, **kwargs: Any) -> str: + return f"{self.object.name} {self.object.ports}" + + def add_directive_header(self, sig: str) -> None: + domain = getattr(self, 'domain', self.objtype) + directive = getattr(self, 'directivetype', 'def') + name = self.format_name() + sourcename = self.get_sourcename() + cell = self.object + + # cell definition + self.add_line(f'.. {domain}:{directive}:: {name}', sourcename) + + # options + opt_attrs = ["title", ] + for attr in opt_attrs: + val = getattr(cell, attr, None) + if val: + self.add_line(f' :{attr}: {val}', sourcename) + + if self.options.noindex: + self.add_line(' :noindex:', sourcename) + + def add_content(self, more_content: Any | None) -> None: + # set sourcename and add content from attribute documentation + sourcename = self.get_sourcename() + startline = int(self.object.source.split(":")[1]) + + for i, line in enumerate(self.object.desc, startline): + self.add_line(line, sourcename, i) + + # add additional content (e.g. from document), if present + if more_content: + for line, src in zip(more_content.data, more_content.items): + self.add_line(line, src[0], src[1]) + + def filter_members( + self, + members: list[tuple[str, Any]], + want_all: bool + ) -> list[tuple[str, Any, bool]]: + return [(x[0], x[1], False) for x in members] + + def get_object_members( + self, + want_all: bool + ) -> tuple[bool, list[tuple[str, Any]]]: + ret: list[tuple[str, str]] = [] + + if self.options.source: + ret.append(('__source', self.real_modname)) + + return False, ret + + def document_members(self, all_members: bool = False) -> None: + want_all = (all_members or + self.options.inherited_members) + # find out which members are documentable + members_check_module, members = self.get_object_members(want_all) + + # document non-skipped members + memberdocumenters: list[tuple[Documenter, bool]] = [] + for (mname, member, isattr) in self.filter_members(members, want_all): + classes = [cls for cls in self.documenters.values() + if cls.can_document_member(member, mname, isattr, self)] + if not classes: + # don't know how to document this member + continue + # prefer the documenter with the highest priority + classes.sort(key=lambda cls: cls.priority) + # give explicitly separated module name, so that members + # of inner classes can be documented + full_mname = self.real_modname + '::' + mname + documenter = classes[-1](self.directive, full_mname, self.indent) + memberdocumenters.append((documenter, isattr)) + + member_order = self.options.member_order or self.config.autodoc_member_order + memberdocumenters = self.sort_members(memberdocumenters, member_order) + + for documenter, isattr in memberdocumenters: + documenter.generate( + all_members=True, real_modname=self.real_modname, + check_module=members_check_module and not isattr) + + def generate( + self, + more_content: Any | None = None, + real_modname: str | None = None, + check_module: bool = False, + all_members: bool = False + ) -> None: + if not self.parse_name(): + # need a cell lib to import from + logger.warning( + f"don't know which cell lib to import for autodocumenting {self.name}", + type = 'cellref' + ) + return + + self.import_object() + + # check __module__ of object (for members not given explicitly) + # if check_module: + # if not self.check_module(): + # return + + sourcename = self.get_sourcename() + self.add_line('', sourcename) + + # format the object's signature, if any + try: + sig = self.format_signature() + except Exception as exc: + logger.warning(('error while formatting signature for %s: %s'), + self.fullname, exc, type='cellref') + return + + # generate the directive header and options, if applicable + self.add_directive_header(sig) + self.add_line('', sourcename) + + # e.g. the module directive doesn't have content + self.indent += self.content_indent + + # add all content (from docstrings, attribute docs etc.) + self.add_content(more_content) + + # document members, if possible + self.document_members(all_members) + +class YosysCellSourceDocumenter(YosysCellDocumenter): + objtype = 'cellsource' + priority = 20 + + @classmethod + def can_document_member( + cls, + member: Any, + membername: str, + isattr: bool, + parent: Any + ) -> bool: + sourcename = str(member).split(":")[0] + if not sourcename.endswith(".v"): + return False + if membername != "__source": + return False + if isinstance(parent, YosysCellDocumenter): + return True + return False + + def add_directive_header(self, sig: str) -> None: + domain = getattr(self, 'domain', 'cell') + directive = getattr(self, 'directivetype', 'source') + name = self.format_name() + sourcename = self.get_sourcename() + cell = self.object + + # cell definition + self.add_line(f'.. {domain}:{directive}:: {name}', sourcename) + + if self.options.linenos: + self.add_line(f' :source: {cell.source.split(":")[0]}', sourcename) + else: + self.add_line(f' :source: {cell.source}', sourcename) + self.add_line(f' :language: verilog', sourcename) + + if self.options.linenos: + startline = int(self.object.source.split(":")[1]) + self.add_line(f' :lineno-start: {startline}', sourcename) + + if self.options.noindex: + self.add_line(' :noindex:', sourcename) + + def add_content(self, more_content: Any | None) -> None: + # set sourcename and add content from attribute documentation + sourcename = self.get_sourcename() + startline = int(self.object.source.split(":")[1]) + + for i, line in enumerate(self.object.code, startline-1): + self.add_line(line, sourcename, i) + + # add additional content (e.g. from document), if present + if more_content: + for line, src in zip(more_content.data, more_content.items): + self.add_line(line, src[0], src[1]) + + def get_object_members( + self, + want_all: bool + ) -> tuple[bool, list[tuple[str, Any]]]: + return False, [] + +def setup(app: Sphinx) -> dict[str, Any]: + app.setup_extension('sphinx.ext.autodoc') + app.add_autodocumenter(YosysCellDocumenter) + app.add_autodocumenter(YosysCellSourceDocumenter) + return { + 'version': '1', + 'parallel_read_safe': True, + } diff --git a/docs/util/cmdref.py b/docs/util/cmdref.py index aac8993889f..df1f3fb21ab 100644 --- a/docs/util/cmdref.py +++ b/docs/util/cmdref.py @@ -1,5 +1,9 @@ # based on https://github.com/ofosos/sphinxrecipes/blob/master/sphinxrecipes/sphinxrecipes.py +from __future__ import annotations + +from docutils import nodes +from docutils.nodes import Node, Element from docutils.parsers.rst import directives from docutils.parsers.rst.states import Inliner from sphinx.application import Sphinx @@ -7,24 +11,59 @@ from sphinx.domains.std import StandardDomain from sphinx.roles import XRefRole from sphinx.directives import ObjectDescription +from sphinx.directives.code import container_wrapper from sphinx.util.nodes import make_refnode from sphinx import addnodes -class CommandNode(ObjectDescription): +class TocNode(ObjectDescription): + def _object_hierarchy_parts(self, sig_node: addnodes.desc_signature) -> tuple[str, ...]: + if 'fullname' not in sig_node: + return () + + modname = sig_node.get('module') + fullname = sig_node['fullname'] + + if modname: + return (modname, *fullname.split('::')) + else: + return tuple(fullname.split('::')) + + def _toc_entry_name(self, sig_node: addnodes.desc_signature) -> str: + if not sig_node.get('_toc_parts'): + return '' + + config = self.env.app.config + objtype = sig_node.parent.get('objtype') + if config.add_function_parentheses and objtype in {'function', 'method'}: + parens = '()' + else: + parens = '' + *parents, name = sig_node['_toc_parts'] + if config.toc_object_entries_show_parents == 'domain': + return sig_node.get('fullname', name) + parens + if config.toc_object_entries_show_parents == 'hide': + return name + parens + if config.toc_object_entries_show_parents == 'all': + return '.'.join(parents + [name + parens]) + return '' + +class CommandNode(TocNode): """A custom node that describes a command.""" name = 'cmd' required_arguments = 1 option_spec = { - 'title': directives.unchanged_required, + 'title': directives.unchanged, 'tags': directives.unchanged } def handle_signature(self, sig, signode: addnodes.desc_signature): + fullname = sig + signode['fullname'] = fullname signode += addnodes.desc_addname(text="yosys> help ") signode += addnodes.desc_name(text=sig) - return sig + return fullname def add_target_and_index(self, name_cls, sig, signode): signode['ids'].append(type(self).name + '-' + sig) @@ -32,7 +71,7 @@ def add_target_and_index(self, name_cls, sig, signode): name = "{}.{}.{}".format(self.name, type(self).__name__, sig) tagmap = self.env.domaindata[type(self).name]['obj2tag'] tagmap[name] = list(self.options.get('tags', '').split(' ')) - title = self.options.get('title') + title = self.options.get('title', sig) titlemap = self.env.domaindata[type(self).name]['obj2title'] titlemap[name] = title objs = self.env.domaindata[type(self).name]['objects'] @@ -48,6 +87,111 @@ class CellNode(CommandNode): name = 'cell' +class CellSourceNode(TocNode): + """A custom code block for including cell source.""" + + name = 'cellsource' + + option_spec = { + "source": directives.unchanged_required, + "language": directives.unchanged_required, + 'lineno-start': int, + } + + def handle_signature( + self, + sig, + signode: addnodes.desc_signature + ) -> str: + language = self.options.get('language') + fullname = sig + "::" + language + signode['fullname'] = fullname + signode += addnodes.desc_name(text="Simulation model") + signode += addnodes.desc_sig_space() + signode += addnodes.desc_addname(text=f'({language})') + return fullname + + def add_target_and_index( + self, + name: str, + sig: str, + signode: addnodes.desc_signature + ) -> None: + idx = f'{".".join(self.name.split(":"))}.{sig}' + signode['ids'].append(idx) + + def run(self) -> list[Node]: + """Override run to parse content as a code block""" + if ':' in self.name: + self.domain, self.objtype = self.name.split(':', 1) + else: + self.domain, self.objtype = '', self.name + self.indexnode = addnodes.index(entries=[]) + + node = addnodes.desc() + node.document = self.state.document + source, line = self.get_source_info() + if line is not None: + line -= 1 + self.state.document.note_source(source, line) + node['domain'] = self.domain + # 'desctype' is a backwards compatible attribute + node['objtype'] = node['desctype'] = self.objtype + node['noindex'] = noindex = ('noindex' in self.options) + node['noindexentry'] = ('noindexentry' in self.options) + node['nocontentsentry'] = ('nocontentsentry' in self.options) + if self.domain: + node['classes'].append(self.domain) + node['classes'].append(node['objtype']) + + self.names = [] + signatures = self.get_signatures() + for sig in signatures: + # add a signature node for each signature in the current unit + # and add a reference target for it + signode = addnodes.desc_signature(sig, '') + self.set_source_info(signode) + node.append(signode) + try: + # name can also be a tuple, e.g. (classname, objname); + # this is strictly domain-specific (i.e. no assumptions may + # be made in this base class) + name = self.handle_signature(sig, signode) + except ValueError: + # signature parsing failed + signode.clear() + signode += addnodes.desc_name(sig, sig) + continue # we don't want an index entry here + finally: + # Private attributes for ToC generation. Will be modified or removed + # without notice. + if self.env.app.config.toc_object_entries: + signode['_toc_parts'] = self._object_hierarchy_parts(signode) + signode['_toc_name'] = self._toc_entry_name(signode) + else: + signode['_toc_parts'] = () + signode['_toc_name'] = '' + if name not in self.names: + self.names.append(name) + if not noindex: + # only add target and index entry if this is the first + # description of the object with this name in this desc block + self.add_target_and_index(name, sig, signode) + + # handle code + code = '\n'.join(self.content) + literal: Element = nodes.literal_block(code, code) + if 'lineno-start' in self.options: + literal['linenos'] = True + literal['highlight_args'] = { + 'linenostart': self.options['lineno-start'] + } + literal['classes'] += self.options.get('class', []) + literal['language'] = self.options.get('language') + literal = container_wrapper(self, literal, self.options.get('source')) + + return [self.indexnode, node, literal] + class TagIndex(Index): """A custom directive that creates a tag matrix.""" @@ -223,6 +367,7 @@ class CellDomain(CommandDomain): directives = { 'def': CellNode, + 'source': CellSourceNode, } indices = {