Add support for python-only projects (#28)

* Add support for python-only projects

This commit:

* Allows `rosdoc2` to exhibit different behavior conditional
  on the currently processed package's build type. Essentially:

  * `ament_cmake`: +`doxygen`/`exhale`/`breathe, -`sphinx-apidoc`
  * `ament_python`: -`doxygen`/`exhale`/`breathe, +`sphinx-apidoc`

* Introduces settings that let users manually configure
  if they want to always run `sphinx-apidoc` or `doxygen` regardless of
  the package's build type (helpful for "mixed" packages).

* Produces toctree entries appropriate for each package, again
  conditional on the build type and the tools invoked by `rosdoc2`.

Signed-off-by: Abrar Rahman Protyasha <[email protected]>
Co-authored-by: Chris Lalancette <[email protected]>
Co-authored-by: William Woodall <[email protected]>
Co-authored-by: R Kent James <[email protected]>

* Address flake8 linter warnings

Signed-off-by: Abrar Rahman Protyasha <[email protected]>

* Generate different index entry based on build type

Signed-off-by: Abrar Rahman Protyasha <[email protected]>

* Log at debug level rather than info level

Signed-off-by: Abrar Rahman Protyasha <[email protected]>
Co-authored-by: Chris Lalancette <[email protected]>

Co-authored-by: Chris Lalancette <[email protected]>

* Make whitespace adjustments in messages

Signed-off-by: Abrar Rahman Protyasha <[email protected]>
Co-authored-by: Chris Lalancette <[email protected]>

Co-authored-by: Chris Lalancette <[email protected]>

* Add the module path to sys.path (not its parent)

Without a trailing slash, `os.path.dirname` returns a path to the
the parent directory of the path provided, which defeats the point
of adding the module path to sys.path.

Signed-off-by: Abrar Rahman Protyasha <[email protected]>

* Only mock dependencies that cannot be imported

Signed-off-by: Abrar Rahman Protyasha <[email protected]>

* Expose enable_autodoc setting to users

Signed-off-by: Abrar Rahman Protyasha <[email protected]>

* Specify expected exception type to satisfy linter

Signed-off-by: Abrar Rahman Protyasha <[email protected]>

---------

Signed-off-by: Abrar Rahman Protyasha <[email protected]>
Co-authored-by: Chris Lalancette <[email protected]>
Co-authored-by: William Woodall <[email protected]>
Co-authored-by: R Kent James <[email protected]>
Co-authored-by: Chris Lalancette <[email protected]>

rosdoc2/verbs/build/build_context.py

@@ -26,3 +26,7 @@ def __init__(self, *, configuration_file_path, package_object, tool_options):
         self.configuration_file_path = configuration_file_path
         self.package = package_object
         self.tool_options = tool_options
+        self.build_type = package_object.get_build_type()
+        self.python_source = None
+        self.always_run_doxygen = False
+        self.always_run_sphinx_apidoc = False

rosdoc2/verbs/build/builders/doxygen_builder.py

@@ -79,6 +79,17 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context):
 
         assert self.builder_type == 'doxygen'
 
+        # If the build type is not `ament_cmake/cmake`, there is no reason
+        # to create a doxygen builder.
+        if (
+            self.build_context.build_type not in
+            ('ament_cmake', 'cmake') and not self.build_context.always_run_doxygen
+        ):
+            logger.debug(
+                f"The package build type is not 'ament_cmake' or 'cmake', hence the "
+                f"'{self.builder_type}' builder was not configured")
+            return
+
         self.doxyfile = None
         self.extra_doxyfile_statements = []
         self.rosdoc2_doxyfile_statements = []
@@ -152,6 +163,16 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context):
 
     def build(self, *, doc_build_folder, output_staging_directory):
         """Actually do the build."""
+        # If the build type is not 'ament_cmake/cmake', there is no reason to run doxygen.
+        if (
+            self.build_context.build_type not in
+            ('ament_cmake', 'cmake') and not self.build_context.always_run_doxygen
+        ):
+            logger.debug(
+                f"The package build type is not 'ament_cmake' or 'cmake', hence the "
+                f"'{self.builder_type}' builder was not invoked")
+            return None  # Explicitly generated no documentation.
+
         # If both doxyfile and doxyfile_content are None, that means there is
         # no reason to run doxygen.
         if self.doxyfile is None and self.doxyfile_content is None:

rosdoc2/verbs/build/builders/sphinx_builder.py

@@ -18,18 +18,41 @@
 import shutil
 import subprocess
 
+import setuptools
+
 from ..builder import Builder
 from ..collect_inventory_files import collect_inventory_files
 from ..create_format_map_from_package import create_format_map_from_package
 
 logger = logging.getLogger('rosdoc2')
 
+
+def generate_package_toc_entry(*, build_context) -> str:
+    """Construct a table of content (toc) entry for the package being processed."""
+    build_type = build_context.build_type
+    always_run_doxygen = build_context.always_run_doxygen
+    always_run_sphinx_apidoc = build_context.always_run_sphinx_apidoc
+    # The TOC entries have to be indented by three (or any N) spaces
+    # inside the string to fall under the `:toctree:` directive
+    toc_entry_cpp = f'{build_context.package.name} <generated/index>\n'
+    toc_entry_py = f'{build_context.package.name} <modules>\n'
+    toc_entry = ''
+
+    if build_type == 'ament_python' or always_run_sphinx_apidoc:
+        toc_entry += toc_entry_py
+    if build_type in ['ament_cmake', 'cmake'] or always_run_doxygen:
+        toc_entry += toc_entry_cpp
+
+    return toc_entry
+
+
 rosdoc2_wrapping_conf_py_template = """\
 ## Generated by rosdoc2.verbs.build.builders.SphinxBuilder.
 ## This conf.py imports the user defined (or default if none was provided)
 ## conf.py, extends the settings to support Breathe and Exhale and to set up
 ## intersphinx mappings correctly, among other things.
 
+import os
 import sys
 
 ## exec the user's conf.py to bring all of their settings into this file.
@@ -49,29 +72,54 @@ def ensure_global(name, default):
     print('[rosdoc2] enabling autodoc', file=sys.stderr)
     extensions.append('sphinx.ext.autodoc')
 
+    pkgs_to_mock = []
+    import importlib
+    for exec_depend in {exec_depends}:
+        try:
+            importlib.import_module(exec_depend)
+        except ImportError:
+            pkgs_to_mock.append(exec_depend)
+    autodoc_mock_imports = pkgs_to_mock
+
 if rosdoc2_settings.get('enable_intersphinx', True):
     print('[rosdoc2] enabling intersphinx', file=sys.stderr)
     extensions.append('sphinx.ext.intersphinx')
 
-if rosdoc2_settings.get('enable_breathe', True):
+build_type = '{build_type}'
+always_run_doxygen = {always_run_doxygen}
+# By default, the `exhale`/`breathe` extensions should be added if `doxygen` was invoked
+is_doxygen_invoked = build_type in ('ament_cmake', 'cmake') or always_run_doxygen
+
+if rosdoc2_settings.get('enable_breathe', is_doxygen_invoked):
     # Configure Breathe.
     # Breathe ingests the XML output from Doxygen and makes it accessible from Sphinx.
     print('[rosdoc2] enabling breathe', file=sys.stderr)
+    # First check that doxygen would have been run
+    if not is_doxygen_invoked:
+        raise RuntimeError(
+            "Cannot enable the 'breathe' extension if 'doxygen' is not invoked. "
+            "Please enable 'always_run_doxygen' if the package is not an "
+            "'ament_cmake' or 'cmake' package.")
     ensure_global('breathe_projects', {{}})
-    breathe_projects.update({{
-{breathe_projects}}})
+    breathe_projects.update({{{breathe_projects}}})
     if breathe_projects:
         # Enable Breathe and arbitrarily select the first project.
         extensions.append('breathe')
         breathe_default_project = next(iter(breathe_projects.keys()))
 
-if rosdoc2_settings.get('enable_exhale', True):
+if rosdoc2_settings.get('enable_exhale', is_doxygen_invoked):
     # Configure Exhale.
     # Exhale uses the output of Doxygen and Breathe to create easier to browse pages
     # for classes and functions documented with Doxygen.
     # This is similar to the class hierarchies and namespace listing provided by
     # Doxygen out of the box.
     print('[rosdoc2] enabling exhale', file=sys.stderr)
+    # First check that doxygen would have been run
+    if not is_doxygen_invoked:
+        raise RuntimeError(
+            "Cannot enable the 'breathe' extension if 'doxygen' is not invoked. "
+            "Please enable 'always_run_doxygen' if the package is not an "
+            "'ament_cmake' or 'cmake' package.")
     extensions.append('exhale')
     ensure_global('exhale_args', {{}})
 
@@ -125,8 +173,6 @@ def ensure_global(name, default):
     print(f"[rosdoc2] adding markdown parser", file=sys.stderr)
     # The `myst_parser` is used specifically if there are markdown files
     # in the sphinx project
-    # TODO(aprotyas): Migrate files under the `include` tag in the project's Doxygen
-    # configuration into the Sphinx project tree used to run the Sphinx builder in.
     extensions.append('myst_parser')
 """  # noqa: W605
 
@@ -240,6 +286,9 @@ def ensure_global(name, default):
     ## If an empty dictionary is provided, breathe defaults will be used.
     # 'exhale_specs_mapping': {{}},
 
+    ## This setting, if True, will ensure autodoc is part of the 'extensions'.
+    # 'enable_autodoc': True,
+
     ## This setting, if True, will ensure intersphinx is part of the 'extensions'.
     # 'enable_intersphinx': True,
 
@@ -266,7 +315,7 @@ def ensure_global(name, default):
 .. toctree::
    :maxdepth: 2
 
-   {package.name} <generated/index>
+   {package_toc_entry}
 
 Indices and Search
 ==================
@@ -333,7 +382,10 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context):
     def build(self, *, doc_build_folder, output_staging_directory):
         """Actually do the build."""
         # Check that doxygen_xml_directory exists relative to output staging, if specified.
-        if self.doxygen_xml_directory is not None:
+        should_run_doxygen = \
+            self.build_context.build_type in ('ament_cmake', 'cmake') or \
+            self.build_context.always_run_doxygen
+        if self.doxygen_xml_directory is not None and should_run_doxygen:
             self.doxygen_xml_directory = \
                 os.path.join(output_staging_directory, self.doxygen_xml_directory)
             self.doxygen_xml_directory = os.path.abspath(self.doxygen_xml_directory)
@@ -379,12 +431,61 @@ def build(self, *, doc_build_folder, output_staging_directory):
             if package_name != self.build_context.package.name
         ]
 
+        package_xml_directory = os.path.dirname(self.build_context.package.filename)
+        # If 'python_source' is specified, construct 'package_src_directory' from it
+        if self.build_context.python_source is not None:
+            package_src_directory = \
+                os.path.abspath(
+                    os.path.join(
+                        package_xml_directory,
+                        self.build_context.python_source))
+        # If not provided, try to find the package source direcotry
+        else:
+            package_list = setuptools.find_packages(where=package_xml_directory)
+            if self.build_context.package.name in package_list:
+                package_src_directory = \
+                    os.path.abspath(
+                        os.path.join(
+                            package_xml_directory,
+                            self.build_context.package.name))
+            else:
+                package_src_directory = None
+
         # Setup rosdoc2 Sphinx file which will include and extend the one in `sourcedir`.
         self.generate_wrapping_rosdoc2_sphinx_project_into_directory(
             doc_build_folder,
             sourcedir,
+            package_src_directory,
             intersphinx_mapping_extensions)
 
+        # If the package has build type `ament_python`, or if the user configured
+        # to run `sphinx-apidoc`, then invoke `sphinx-apidoc` before building
+        if (
+            self.build_context.build_type == 'ament_python'
+            or self.build_context.always_run_sphinx_apidoc
+        ):
+
+            if not package_src_directory or not os.path.isdir(package_src_directory):
+                raise RuntimeError(
+                    'Could not locate source directory to invoke sphinx-apidoc in. '
+                    'If this is package does not have a standard Python package layout, '
+                    "please specify the Python source in 'rosdoc2.yaml'.")
+            cmd = [
+                'sphinx-apidoc',
+                '-o', os.path.relpath(sourcedir, start=doc_build_folder),
+                '-e',  # Document each module in its own page.
+                package_src_directory,
+            ]
+            logger.info(
+                f"Running sphinx-apidoc: '{' '.join(cmd)}' in '{doc_build_folder}'"
+            )
+            completed_process = subprocess.run(cmd, cwd=doc_build_folder)
+            msg = f"sphinx-apidoc exited with return code '{completed_process.returncode}'"
+            if completed_process.returncode == 0:
+                logger.debug(msg)
+            else:
+                raise RuntimeError(msg)
+
         # Invoke Sphinx-build.
         working_directory = doc_build_folder
         sphinx_output_dir = os.path.abspath(os.path.join(doc_build_folder, 'sphinx_output'))
@@ -474,7 +575,8 @@ def generate_default_project_into_directory(self, directory):
         root_title = f'Welcome to the documentation for {package.name}'
         template_variables.update({
             'root_title': root_title,
-            'root_title_underline': '=' * len(root_title)
+            'root_title_underline': '=' * len(root_title),
+            'package_toc_entry': generate_package_toc_entry(build_context=self.build_context)
         })
 
         with open(os.path.join(directory, 'index.rst'), 'w+') as f:
@@ -484,6 +586,7 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory(
         self,
         directory,
         user_sourcedir,
+        package_src_directory,
         intersphinx_mapping_extensions,
     ):
         """Generate the rosdoc2 sphinx project configuration files."""
@@ -496,6 +599,10 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory(
                 f'        "{package.name} Doxygen Project": "{self.doxygen_xml_directory}"')
         template_variables = {
             'package_name': package.name,
+            'package_src_directory': package_src_directory,
+            'exec_depends': [exec_depend.name for exec_depend in package.exec_depends],
+            'build_type': self.build_context.build_type,
+            'always_run_doxygen': self.build_context.always_run_doxygen,
             'user_sourcedir': os.path.abspath(user_sourcedir),
             'user_conf_py_filename': os.path.abspath(os.path.join(user_sourcedir, 'conf.py')),
             'breathe_projects': ',\n'.join(breathe_projects) + '\n    ',

rosdoc2/verbs/build/inspect_package_for_settings.py

@@ -44,6 +44,23 @@
 
     ## If this is not specified explicitly, it defaults to 'true'.
     generate_package_index: true
+
+    ## This setting is relevant mostly if the standard Python package layout cannot
+    ## be assumed for 'sphinx-apidoc' invocation. The user can provide the path
+    ## (relative to the 'package.xml' file) where the Python modules defined by this
+    ## package are located.
+    python_source: '{package_name}'
+
+    ## This setting, if true, attempts to run `doxygen` and the `breathe`/`exhale`
+    ## extensions to `sphinx` regardless of build type. This is most useful if the
+    ## user would like to generate C/C++ API documentation for a package that is not
+    ## of the `ament_cmake/cmake` build type.
+    always_run_doxygen: false
+
+    ## This setting, if true, attempts to run `sphinx-apidoc` regardless of build
+    ## type. This is most useful if the user would like to generate Python API
+    ## documentation for a package that is not of the `ament_python` build type.
+    always_run_sphinx_apidoc: false
 builders:
     ## Each stanza represents a separate build step, performed by a specific 'builder'.
     ## The key of each stanza is the builder to use; this must be one of the

rosdoc2/verbs/build/parse_rosdoc2_yaml.py

@@ -57,6 +57,10 @@ def parse_rosdoc2_yaml(yaml_string, build_context):
             f'expected a dict{{output_dir: build_settings, ...}}, '
             f"got a '{type(settings_dict)}' instead")
 
+    build_context.python_source = settings_dict.get('python_source', None)
+    build_context.always_run_doxygen = settings_dict.get('always_run_doxygen', False)
+    build_context.always_run_sphinx_apidoc = settings_dict.get('always_run_sphinx_apidoc', False)
+
     if 'builders' not in config:
         raise ValueError(
             f"Error parsing file '{file_name}', in the second section, "