From 8c0e7858222f5561dbcfd28a64888cb1376b0ac5 Mon Sep 17 00:00:00 2001 From: R Kent James Date: Thu, 14 Nov 2024 16:29:39 -0800 Subject: [PATCH] Add --yaml-extend option to allow modifying rosdoc2.yaml (#151) * Add --yaml-extend option to allow modifying rosdoc2.yaml Signed-off-by: R. Kent James --- rosdoc2/verbs/build/impl.py | 5 ++ .../build/inspect_package_for_settings.py | 68 ++++++++++++++++- rosdoc2/verbs/build/parse_rosdoc2_yaml.py | 20 +---- test/ex_test.yaml | 76 +++++++++++++++++++ .../funny_docs/funny_doc.rst | 4 + test/test_builder.py | 36 +++++++-- 6 files changed, 182 insertions(+), 27 deletions(-) create mode 100644 test/ex_test.yaml create mode 100644 test/packages/invalid_python_source/funny_docs/funny_doc.rst diff --git a/rosdoc2/verbs/build/impl.py b/rosdoc2/verbs/build/impl.py index cef9d930..8eff6a1c 100644 --- a/rosdoc2/verbs/build/impl.py +++ b/rosdoc2/verbs/build/impl.py @@ -98,6 +98,11 @@ def prepare_arguments(parser): action='store_true', help='enable more output to debug problems' ) + parser.add_argument( + '--yaml-extend', + '-y', + help='Extend rosdoc2.yaml' + ) return parser diff --git a/rosdoc2/verbs/build/inspect_package_for_settings.py b/rosdoc2/verbs/build/inspect_package_for_settings.py index c95fb3c8..ca620e9b 100644 --- a/rosdoc2/verbs/build/inspect_package_for_settings.py +++ b/rosdoc2/verbs/build/inspect_package_for_settings.py @@ -12,12 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os +import yaml + from .build_context import BuildContext +from .builders import create_builder_by_name from .create_format_map_from_package import create_format_map_from_package from .parse_rosdoc2_yaml import parse_rosdoc2_yaml +logger = logging.getLogger('rosdoc2') + DEFAULT_ROSDOC_CONFIG_FILE = """\ ## Default configuration, generated by rosdoc2. @@ -139,5 +145,65 @@ def inspect_package_for_settings(package, tool_options): for depends in package['buildtool_depends']: if str(depends) == 'ament_cmake_python': build_context.ament_cmake_python = True + configs = list(yaml.load_all(rosdoc_config_file, Loader=yaml.SafeLoader)) + + (settings_dict, builders_list) = parse_rosdoc2_yaml(configs, build_context) - return parse_rosdoc2_yaml(rosdoc_config_file, build_context) + # Extend rosdoc2.yaml if desired + # + # An optional fie may be used to modify the values in rosdoc2.yaml for this package. The format + # of this file is as follows: + """ +--- +: + packages: + <1st package name>: + + <2nd package name>: + + + packages: + + + """ + yaml_extend = tool_options.yaml_extend + if yaml_extend: + if not os.path.isfile(yaml_extend): + raise ValueError( + f"yaml_extend path '{yaml_extend}' is not a file") + with open(yaml_extend, 'r') as f: + yaml_extend_text = f.read() + extended_settings = yaml.load(yaml_extend_text, Loader=yaml.SafeLoader) + for ex_name in extended_settings: + if package.name in extended_settings[ex_name]['packages']: + extended_object = extended_settings[ex_name]['packages'][package.name] + if 'settings' in extended_object: + for key, value in extended_object['settings'].items(): + settings_dict[key] = value + logger.info(f'Overriding rosdoc2.yaml setting <{key}> with <{value}>') + if 'builders' in extended_object: + for ex_builder in extended_object['builders']: + ex_builder_name = next(iter(ex_builder)) + # find this object in the builders list + for user_builder in builders_list: + user_builder_name = next(iter(user_builder)) + if user_builder_name == ex_builder_name: + for builder_k, builder_v in ex_builder[ex_builder_name].items(): + logger.info(f'Overriding rosdoc2 builder <{ex_builder_name}> ' + f'property <{builder_k}> with <{builder_v}>') + user_builder[user_builder_name][builder_k] = builder_v + + # if None, python_source is set to either './' or 'src/' + build_context.python_source = settings_dict.get('python_source', None) + build_context.always_run_doxygen = settings_dict.get('always_run_doxygen', False) + build_context.always_run_sphinx_apidoc = settings_dict.get('always_run_sphinx_apidoc', False) + build_context.build_type = settings_dict.get('override_build_type', build_context.build_type) + + builders = [] + for builder in builders_list: + builder_name = next(iter(builder)) + builders.append(create_builder_by_name(builder_name, + builder_dict=builder[builder_name], + build_context=build_context)) + + return (settings_dict, builders) diff --git a/rosdoc2/verbs/build/parse_rosdoc2_yaml.py b/rosdoc2/verbs/build/parse_rosdoc2_yaml.py index e74f5707..5f893d49 100644 --- a/rosdoc2/verbs/build/parse_rosdoc2_yaml.py +++ b/rosdoc2/verbs/build/parse_rosdoc2_yaml.py @@ -12,19 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import yaml -from .builders import create_builder_by_name - - -def parse_rosdoc2_yaml(yaml_string, build_context): +def parse_rosdoc2_yaml(configs, build_context): """ Parse a rosdoc2.yaml configuration string, returning it as a tuple of settings and builders. :return: a tuple with the first item being the tool settings as a dictionary, and the second item being a list of Builder objects. """ - configs = list(yaml.load_all(yaml_string, Loader=yaml.SafeLoader)) file_name = build_context.configuration_file_path if len(configs) != 2: raise ValueError( @@ -57,12 +52,6 @@ def parse_rosdoc2_yaml(yaml_string, build_context): f'expected a dict{{output_dir: build_settings, ...}}, ' f"got a '{type(settings_dict)}' instead") - # if None, python_source is set to either './' or 'src/' - build_context.python_source = settings_dict.get('python_source', None) - build_context.always_run_doxygen = settings_dict.get('always_run_doxygen', False) - build_context.always_run_sphinx_apidoc = settings_dict.get('always_run_sphinx_apidoc', False) - build_context.build_type = settings_dict.get('override_build_type', build_context.build_type) - if 'builders' not in config: raise ValueError( f"Error parsing file '{file_name}', in the second section, " @@ -74,15 +63,10 @@ def parse_rosdoc2_yaml(yaml_string, build_context): 'expected a list of builders, ' f"got a '{type(builders_list)}' instead") - builders = [] for builder in builders_list: if len(builder) != 1: raise ValueError( f"Error parsing file '{file_name}', in the second section, each builder " 'must have exactly one key (which is the type of builder to use)') - builder_name = next(iter(builder)) - builders.append(create_builder_by_name(builder_name, - builder_dict=builder[builder_name], - build_context=build_context)) - return (settings_dict, builders) + return (settings_dict, builders_list) diff --git a/test/ex_test.yaml b/test/ex_test.yaml new file mode 100644 index 00000000..29799a11 --- /dev/null +++ b/test/ex_test.yaml @@ -0,0 +1,76 @@ +--- +docs_support: + packages: + invalid_python_source: + builders: + - sphinx: + user_doc_dir: funny_docs +python_location: + packages: + src_alt_python: + settings: + python_source: launch +default_rosdoc2_yaml: + packages: + empty_doc_dir: + settings: { + ## This setting is relevant mostly if the standard Python package layout cannot + ## be assumed for 'sphinx-apidoc' invocation. The user can provide the path + ## (relative to the 'package.xml' file) where the Python modules defined by this + ## package are located. + python_source: 'empty_doc_dir', + + ## This setting, if true, attempts to run `doxygen` and the `breathe`/`exhale` + ## extensions to `sphinx` regardless of build type. This is most useful if the + ## user would like to generate C/C++ API documentation for a package that is not + ## of the `ament_cmake/cmake` build type. + always_run_doxygen: false, + + ## This setting, if true, attempts to run `sphinx-apidoc` regardless of build + ## type. This is most useful if the user would like to generate Python API + ## documentation for a package that is not of the `ament_python` build type. + always_run_sphinx_apidoc: false, + + ## This setting, if provided, will override the build_type of this package + ## for documentation purposes only. If not provided, documentation will be + ## generated assuming the build_type in package.xml. + override_build_type: 'ament_cmake', + } + builders: + ## Each stanza represents a separate build step, performed by a specific 'builder'. + ## The key of each stanza is the builder to use; this must be one of the + ## available builders. + ## The value of each stanza is a dictionary of settings for the builder that + ## outputs to that directory. + ## Keys in all settings dictionary are: + ## * 'output_dir' - determines output subdirectory for builder instance + ## relative to --output-directory + ## * 'name' - used when referencing the built docs from the index. + + - doxygen: { + name: 'empty_doc_dir Public C/C++ API', + output_dir: 'generated/doxygen', + ## file name for a user-supplied Doxyfile + doxyfile: null, + ## additional statements to add to the Doxyfile, list of strings + extra_doxyfile_statements: [], + } + - sphinx: { + name: 'empty_doc_dir', + ## This path is relative to output staging. + doxygen_xml_directory: 'generated/doxygen/xml', + output_dir: '', + ## If sphinx_sourcedir is specified and not null, then the documentation in that folder + ## (specified relative to the package.xml directory) will replace rosdoc2's normal output. + ## If sphinx_sourcedir is left unspecified, any documentation found in the doc/ or + ## doc/source/ folder will still be included by default, along with other relevant package + ## information. + sphinx_sourcedir: null, + ## Directory (relative to the package.xml directory) where user documentation is found. If + ## documentation is in one of the standard locations (doc/ or doc/source) this is not + ## needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2 + ## output, but includes this user documentation along with other items included by default + ## by rosdoc2. + user_doc_dir: 'doc' + } + diff --git a/test/packages/invalid_python_source/funny_docs/funny_doc.rst b/test/packages/invalid_python_source/funny_docs/funny_doc.rst new file mode 100644 index 00000000..99428dea --- /dev/null +++ b/test/packages/invalid_python_source/funny_docs/funny_doc.rst @@ -0,0 +1,4 @@ +This is in a funny place +======================== + +blah, blah diff --git a/test/test_builder.py b/test/test_builder.py index e70fc5f7..0047cf52 100644 --- a/test/test_builder.py +++ b/test/test_builder.py @@ -35,19 +35,22 @@ def module_dir(tmp_path_factory): return tmp_path_factory.getbasetemp() -def do_build_package(package_path, work_path) -> None: +def do_build_package(package_path, work_path, with_extension=False) -> None: build_dir = work_path / 'build' output_dir = work_path / 'output' cr_dir = work_path / 'cross_references' # Create a top level parser parser = prepare_arguments(argparse.ArgumentParser()) - options = parser.parse_args([ + args = [ '-p', str(package_path), '-c', str(cr_dir), '-o', str(output_dir), '-d', str(build_dir), - ]) + ] + if with_extension: + args.extend(['-y', str(pathlib.Path('test') / 'ex_test.yaml')]) + options = parser.parse_args(args) logger.info(f'*** Building package(s) at {package_path} with options {options}') # run rosdoc2 on the package @@ -93,7 +96,7 @@ def test_full_package(module_dir): def test_default_yaml(module_dir): """Test a package with C++, python, and docs using specified default rosdoc2.yaml configs.""" PKG_NAME = 'default_yaml' - do_build_package(DATAPATH / PKG_NAME, module_dir) + do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True) do_test_full_package(module_dir, pkg_name=PKG_NAME) @@ -101,7 +104,8 @@ def test_default_yaml(module_dir): def test_only_python(module_dir): """Test a pure python package.""" PKG_NAME = 'only_python' - do_build_package(DATAPATH / PKG_NAME, module_dir) + # Use with_extension=True to show that nothing changes if the package is not there. + do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True) includes = [ PKG_NAME, @@ -151,10 +155,13 @@ def test_false_python(module_dir): def test_invalid_python_source(module_dir): PKG_NAME = 'invalid_python_source' - do_build_package(DATAPATH / PKG_NAME, module_dir) + do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True) excludes = ['python api'] - includes = ['This packages incorrectly specifies python source'] + includes = [ + 'This packages incorrectly specifies python source', + 'this is in a funny place', # Documentation found using extended yaml + ] do_test_package(PKG_NAME, module_dir, includes=includes, @@ -231,8 +238,10 @@ def test_has_sphinx_sourcedir(module_dir): def test_empty_doc_dir(module_dir): + # This package is run with an extended rosdoc2.yaml setting that adds all of the + # default rosdoc2.yaml settings to the extended yaml. PKG_NAME = 'empty_doc_dir' - do_build_package(DATAPATH / PKG_NAME, module_dir) + do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True) includes = [ 'package with an empty doc directory', # The package description @@ -250,3 +259,14 @@ def test_empty_doc_dir(module_dir): links_exist=links_exist) do_test_package(PKG_NAME, module_dir) + + +def test_src_alt_python(module_dir): + PKG_NAME = 'src_alt_python' + do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True) + + includes = ['python api'] # We found the python source with the extended yaml + links_exist = ['dummy.html'] # We found python source with extended yaml + do_test_package(PKG_NAME, module_dir, + includes=includes, + links_exist=links_exist)