Add --yaml-extend option to allow modifying rosdoc2.yaml (#151)

* Add --yaml-extend option to allow modifying rosdoc2.yaml

Signed-off-by: R. Kent James <[email protected]>

rosdoc2/verbs/build/impl.py

@@ -98,6 +98,11 @@ def prepare_arguments(parser):
         action='store_true',
         help='enable more output to debug problems'
     )
+    parser.add_argument(
+        '--yaml-extend',
+        '-y',
+        help='Extend rosdoc2.yaml'
+    )
     return parser
 
 

rosdoc2/verbs/build/inspect_package_for_settings.py

@@ -12,12 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import os
 
+import yaml
+
 from .build_context import BuildContext
+from .builders import create_builder_by_name
 from .create_format_map_from_package import create_format_map_from_package
 from .parse_rosdoc2_yaml import parse_rosdoc2_yaml
 
+logger = logging.getLogger('rosdoc2')
+
 DEFAULT_ROSDOC_CONFIG_FILE = """\
 ## Default configuration, generated by rosdoc2.
 
@@ -139,5 +145,65 @@ def inspect_package_for_settings(package, tool_options):
     for depends in package['buildtool_depends']:
         if str(depends) == 'ament_cmake_python':
             build_context.ament_cmake_python = True
+    configs = list(yaml.load_all(rosdoc_config_file, Loader=yaml.SafeLoader))
+
+    (settings_dict, builders_list) = parse_rosdoc2_yaml(configs, build_context)
 
-    return parse_rosdoc2_yaml(rosdoc_config_file, build_context)
+    # Extend rosdoc2.yaml if desired
+    #
+    # An optional fie may be used to modify the values in rosdoc2.yaml for this package. The format
+    # of this file is as follows:
+    """
+---
+<some_identifier_describing_a_collection_of_packages>:
+    packages:
+        <1st package name>:
+            <anything valid in rosdoc2.yaml file>
+        <2nd package name>:
+            <more valid rosdoc2.yaml>
+<another_description>
+    packages:
+        <another_package_name>
+            <valid rosdoc2.yaml>
+    """
+    yaml_extend = tool_options.yaml_extend
+    if yaml_extend:
+        if not os.path.isfile(yaml_extend):
+            raise ValueError(
+                f"yaml_extend path '{yaml_extend}' is not a file")
+        with open(yaml_extend, 'r') as f:
+            yaml_extend_text = f.read()
+        extended_settings = yaml.load(yaml_extend_text, Loader=yaml.SafeLoader)
+        for ex_name in extended_settings:
+            if package.name in extended_settings[ex_name]['packages']:
+                extended_object = extended_settings[ex_name]['packages'][package.name]
+                if 'settings' in extended_object:
+                    for key, value in extended_object['settings'].items():
+                        settings_dict[key] = value
+                        logger.info(f'Overriding rosdoc2.yaml setting  <{key}> with <{value}>')
+                if 'builders' in extended_object:
+                    for ex_builder in extended_object['builders']:
+                        ex_builder_name = next(iter(ex_builder))
+                        # find this object in the builders list
+                        for user_builder in builders_list:
+                            user_builder_name = next(iter(user_builder))
+                            if user_builder_name == ex_builder_name:
+                                for builder_k, builder_v in ex_builder[ex_builder_name].items():
+                                    logger.info(f'Overriding rosdoc2 builder <{ex_builder_name}> '
+                                                f'property <{builder_k}> with <{builder_v}>')
+                                    user_builder[user_builder_name][builder_k] = builder_v
+
+    # if None, python_source is set to either './<package.name>' or 'src/<package.name>'
+    build_context.python_source = settings_dict.get('python_source', None)
+    build_context.always_run_doxygen = settings_dict.get('always_run_doxygen', False)
+    build_context.always_run_sphinx_apidoc = settings_dict.get('always_run_sphinx_apidoc', False)
+    build_context.build_type = settings_dict.get('override_build_type', build_context.build_type)
+
+    builders = []
+    for builder in builders_list:
+        builder_name = next(iter(builder))
+        builders.append(create_builder_by_name(builder_name,
+                                               builder_dict=builder[builder_name],
+                                               build_context=build_context))
+
+    return (settings_dict, builders)

rosdoc2/verbs/build/parse_rosdoc2_yaml.py

@@ -12,19 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import yaml
 
-from .builders import create_builder_by_name
-
-
-def parse_rosdoc2_yaml(yaml_string, build_context):
+def parse_rosdoc2_yaml(configs, build_context):
     """
     Parse a rosdoc2.yaml configuration string, returning it as a tuple of settings and builders.
 
     :return: a tuple with the first item being the tool settings as a dictionary,
         and the second item being a list of Builder objects.
     """
-    configs = list(yaml.load_all(yaml_string, Loader=yaml.SafeLoader))
     file_name = build_context.configuration_file_path
     if len(configs) != 2:
         raise ValueError(
@@ -57,12 +52,6 @@ def parse_rosdoc2_yaml(yaml_string, build_context):
             f'expected a dict{{output_dir: build_settings, ...}}, '
             f"got a '{type(settings_dict)}' instead")
 
-    # if None, python_source is set to either './<package.name>' or 'src/<package.name>'
-    build_context.python_source = settings_dict.get('python_source', None)
-    build_context.always_run_doxygen = settings_dict.get('always_run_doxygen', False)
-    build_context.always_run_sphinx_apidoc = settings_dict.get('always_run_sphinx_apidoc', False)
-    build_context.build_type = settings_dict.get('override_build_type', build_context.build_type)
-
     if 'builders' not in config:
         raise ValueError(
             f"Error parsing file '{file_name}', in the second section, "
@@ -74,15 +63,10 @@ def parse_rosdoc2_yaml(yaml_string, build_context):
             'expected a list of builders, '
             f"got a '{type(builders_list)}' instead")
 
-    builders = []
     for builder in builders_list:
         if len(builder) != 1:
             raise ValueError(
                 f"Error parsing file '{file_name}', in the second section, each builder "
                 'must have exactly one key (which is the type of builder to use)')
-        builder_name = next(iter(builder))
-        builders.append(create_builder_by_name(builder_name,
-                                               builder_dict=builder[builder_name],
-                                               build_context=build_context))
 
-    return (settings_dict, builders)
+    return (settings_dict, builders_list)

test/ex_test.yaml

@@ -0,0 +1,76 @@
+---
+docs_support:
+  packages:
+    invalid_python_source:
+      builders:
+        - sphinx:
+              user_doc_dir: funny_docs
+python_location:
+  packages:
+    src_alt_python:
+      settings:
+        python_source: launch
+default_rosdoc2_yaml:
+  packages:
+    empty_doc_dir:
+      settings: {
+          ## This setting is relevant mostly if the standard Python package layout cannot
+          ## be assumed for 'sphinx-apidoc' invocation. The user can provide the path
+          ## (relative to the 'package.xml' file) where the Python modules defined by this
+          ## package are located.
+          python_source: 'empty_doc_dir',
+
+          ## This setting, if true, attempts to run `doxygen` and the `breathe`/`exhale`
+          ## extensions to `sphinx` regardless of build type. This is most useful if the
+          ## user would like to generate C/C++ API documentation for a package that is not
+          ## of the `ament_cmake/cmake` build type.
+          always_run_doxygen: false,
+
+          ## This setting, if true, attempts to run `sphinx-apidoc` regardless of build
+          ## type. This is most useful if the user would like to generate Python API
+          ## documentation for a package that is not of the `ament_python` build type.
+          always_run_sphinx_apidoc: false,
+
+          ## This setting, if provided, will override the build_type of this package
+          ## for documentation purposes only. If not provided, documentation will be
+          ## generated assuming the build_type in package.xml.
+          override_build_type: 'ament_cmake',
+      }
+      builders:
+          ## Each stanza represents a separate build step, performed by a specific 'builder'.
+          ## The key of each stanza is the builder to use; this must be one of the
+          ## available builders.
+          ## The value of each stanza is a dictionary of settings for the builder that
+          ## outputs to that directory.
+          ## Keys in all settings dictionary are:
+          ##  * 'output_dir' - determines output subdirectory for builder instance
+          ##                   relative to --output-directory
+          ##  * 'name' - used when referencing the built docs from the index.
+
+          - doxygen: {
+              name: 'empty_doc_dir Public C/C++ API',
+              output_dir: 'generated/doxygen',
+              ## file name for a user-supplied Doxyfile
+              doxyfile: null,
+              ## additional statements to add to the Doxyfile, list of strings
+              extra_doxyfile_statements: [],
+            }
+          - sphinx: {
+              name: 'empty_doc_dir',
+              ## This path is relative to output staging.
+              doxygen_xml_directory: 'generated/doxygen/xml',
+              output_dir: '',
+              ## If sphinx_sourcedir is specified and not null, then the documentation in that folder
+              ## (specified relative to the package.xml directory) will replace rosdoc2's normal output.
+              ## If sphinx_sourcedir is left unspecified, any documentation found in the doc/ or
+              ## doc/source/ folder will still be included by default, along with other relevant package
+              ## information.
+              sphinx_sourcedir: null,
+              ## Directory (relative to the package.xml directory) where user documentation is found. If
+              ## documentation is in one of the standard locations (doc/ or doc/source) this is not
+              ## needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2
+              ## output, but includes this user documentation along with other items included by default
+              ## by rosdoc2.
+              user_doc_dir: 'doc'
+            }
+

test/packages/invalid_python_source/funny_docs/funny_doc.rst

@@ -0,0 +1,4 @@
+This is in a funny place
+========================
+
+blah, blah

test/test_builder.py

@@ -35,19 +35,22 @@ def module_dir(tmp_path_factory):
     return tmp_path_factory.getbasetemp()
 
 
-def do_build_package(package_path, work_path) -> None:
+def do_build_package(package_path, work_path, with_extension=False) -> None:
     build_dir = work_path / 'build'
     output_dir = work_path / 'output'
     cr_dir = work_path / 'cross_references'
 
     # Create a top level parser
     parser = prepare_arguments(argparse.ArgumentParser())
-    options = parser.parse_args([
+    args = [
         '-p', str(package_path),
         '-c', str(cr_dir),
         '-o', str(output_dir),
         '-d', str(build_dir),
-    ])
+    ]
+    if with_extension:
+        args.extend(['-y', str(pathlib.Path('test') / 'ex_test.yaml')])
+    options = parser.parse_args(args)
     logger.info(f'*** Building package(s) at {package_path} with options {options}')
 
     # run rosdoc2 on the package
@@ -93,15 +96,16 @@ def test_full_package(module_dir):
 def test_default_yaml(module_dir):
     """Test a package with C++, python, and docs using specified default rosdoc2.yaml configs."""
     PKG_NAME = 'default_yaml'
-    do_build_package(DATAPATH / PKG_NAME, module_dir)
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
 
     do_test_full_package(module_dir, pkg_name=PKG_NAME)
 
 
 def test_only_python(module_dir):
     """Test a pure python package."""
     PKG_NAME = 'only_python'
-    do_build_package(DATAPATH / PKG_NAME, module_dir)
+    # Use with_extension=True to show that nothing changes if the package is not there.
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
 
     includes = [
         PKG_NAME,
@@ -151,10 +155,13 @@ def test_false_python(module_dir):
 
 def test_invalid_python_source(module_dir):
     PKG_NAME = 'invalid_python_source'
-    do_build_package(DATAPATH / PKG_NAME, module_dir)
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
 
     excludes = ['python api']
-    includes = ['This packages incorrectly specifies python source']
+    includes = [
+        'This packages incorrectly specifies python source',
+        'this is in a funny place',  # Documentation found using extended yaml
+    ]
 
     do_test_package(PKG_NAME, module_dir,
                     includes=includes,
@@ -231,8 +238,10 @@ def test_has_sphinx_sourcedir(module_dir):
 
 
 def test_empty_doc_dir(module_dir):
+    # This package is run with an extended rosdoc2.yaml setting that adds all of the
+    # default rosdoc2.yaml settings to the extended yaml.
     PKG_NAME = 'empty_doc_dir'
-    do_build_package(DATAPATH / PKG_NAME, module_dir)
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
 
     includes = [
         'package with an empty doc directory',  # The package description
@@ -250,3 +259,14 @@ def test_empty_doc_dir(module_dir):
                     links_exist=links_exist)
 
     do_test_package(PKG_NAME, module_dir)
+
+
+def test_src_alt_python(module_dir):
+    PKG_NAME = 'src_alt_python'
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
+
+    includes = ['python api']  # We found the python source with the extended yaml
+    links_exist = ['dummy.html']  # We found python source with extended yaml
+    do_test_package(PKG_NAME, module_dir,
+                    includes=includes,
+                    links_exist=links_exist)