From 78daadc93b9ae9d9fa36dba398a9b4d41bf0a53e Mon Sep 17 00:00:00 2001 From: "R. Kent James" Date: Thu, 8 Aug 2024 14:36:24 -0700 Subject: [PATCH 1/3] Don't require conf.py if sphinx_sourcedir is specified --- .../verbs/build/builders/sphinx_builder.py | 96 ++++++++----------- rosdoc2/verbs/build/include_user_docs.py | 20 ++-- test/packages/false_python/docs/somedocs.md | 3 + test/packages/false_python/package.xml | 1 + test/packages/false_python/rosdoc2.yaml | 59 ++++++++++++ test/test_builder.py | 9 +- 6 files changed, 122 insertions(+), 66 deletions(-) create mode 100644 test/packages/false_python/docs/somedocs.md create mode 100644 test/packages/false_python/rosdoc2.yaml diff --git a/rosdoc2/verbs/build/builders/sphinx_builder.py b/rosdoc2/verbs/build/builders/sphinx_builder.py index 657d7179..5382c90d 100644 --- a/rosdoc2/verbs/build/builders/sphinx_builder.py +++ b/rosdoc2/verbs/build/builders/sphinx_builder.py @@ -399,6 +399,8 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context): raise RuntimeError( f"Error Sphinx SOURCEDIR '{value}' does not exist relative " f"to '{configuration_file_path}', or is not a directory.") + else: + logger.info(f'The user specified sphinx_sourcedir as {sphinx_sourcedir}') self.sphinx_sourcedir = sphinx_sourcedir else: raise RuntimeError(f"Error the Sphinx builder does not support key '{key}'") @@ -475,10 +477,6 @@ def build(self, *, doc_build_folder, output_staging_directory): generate_standard_document_files(standard_docs, wrapped_sphinx_directory) logger.info(f'standard_docs: {standard_docs}') - # include user documentation - doc_directories = include_user_docs(package_xml_directory, wrapped_sphinx_directory) - logger.info(f'doc_directories: {doc_directories}') - # Check if the user provided a sphinx directory. sphinx_project_directory = self.sphinx_sourcedir if sphinx_project_directory is not None: @@ -488,23 +486,18 @@ def build(self, *, doc_build_folder, output_staging_directory): f"'{sphinx_project_directory}' will be used.") else: # If the user does not supply a Sphinx sourcedir, check the standard locations. - standard_sphinx_sourcedir = self.locate_sphinx_sourcedir_from_standard_locations() - if standard_sphinx_sourcedir is not None: + sphinx_project_directory = self.locate_sphinx_sourcedir_from_standard_locations() + if sphinx_project_directory is not None: logger.info( 'Note: no sourcedir provided, but a Sphinx sourcedir located in the ' - f"standard location '{standard_sphinx_sourcedir}' and that will be used.") - sphinx_project_directory = standard_sphinx_sourcedir - else: - # If the user does not supply a Sphinx sourcedir, and there is no Sphinx project - # in the conventional location, i.e. '/doc', create a temporary - # Sphinx project in the doc build directory to enable cross-references. - logger.info( - 'Note: no sourcedir provided by the user and no Sphinx sourcedir was found ' - 'in the standard locations, therefore using a default Sphinx configuration.') - sphinx_project_directory = os.path.join(doc_build_folder, 'default_sphinx_project') + f"standard location '{sphinx_project_directory}' and that will be used.") - self.generate_default_project_into_directory( - sphinx_project_directory, python_src_directory) + # include user documentation + if sphinx_project_directory: + doc_directories = include_user_docs(sphinx_project_directory, wrapped_sphinx_directory) + logger.info(f'doc_directories: {doc_directories}') + else: + doc_directories = None # Collect intersphinx mapping extensions from discovered inventory files. inventory_files = \ @@ -534,11 +527,22 @@ def build(self, *, doc_build_folder, output_staging_directory): 'package': self.build_context.package, }) + # If the user did no include a conf.py in sphinx_project_directory, generate + # a default conf.py + conf_py_directory = sphinx_project_directory + if not conf_py_directory or not os.path.isfile(os.path.join(conf_py_directory, 'conf.py')): + logger.info('Note: no conf.py provided by the user, ' + 'therefore using a default Sphinx configuration.') + conf_py_directory = os.path.join(doc_build_folder, 'default_sphinx_project') + os.makedirs(conf_py_directory, exist_ok=True) + self.generate_default_project_into_directory( + conf_py_directory, python_src_directory) + # Setup rosdoc2 Sphinx file which will include and extend the one in # `sphinx_project_directory`. self.generate_wrapping_rosdoc2_sphinx_project_into_directory( wrapped_sphinx_directory, - sphinx_project_directory, + conf_py_directory, python_src_directory, intersphinx_mapping_extensions) @@ -621,8 +625,8 @@ def locate_sphinx_sourcedir_from_standard_locations(self): If the sphinx configuration exists in a standard location, return it, otherwise return None. The standard locations are - '/doc/source/conf.py' and - '/doc/conf.py', for projects that selected + '/doc/source' and + '/doc', for projects that selected "separate source and build directories" when running Sphinx-quickstart and those that did not, respectively. """ @@ -632,15 +636,13 @@ def locate_sphinx_sourcedir_from_standard_locations(self): os.path.join(package_xml_directory, 'doc', 'source'), ] for option in options: - if os.path.isfile(os.path.join(option, 'conf.py')): + if os.path.isdir(option): return option return None def generate_default_project_into_directory( - self, sphinx_project_directory, python_src_directory): - """Generate the default project configuration files.""" - os.makedirs(sphinx_project_directory, exist_ok=True) - + self, conf_py_directory, python_src_directory): + """Generate the default project configuration files if needed.""" package = self.build_context.package self.template_variables.update({ 'package': package, @@ -652,45 +654,31 @@ def generate_default_project_into_directory( )), }) - with open(os.path.join(sphinx_project_directory, 'conf.py'), 'w+') as f: + with open(os.path.join(conf_py_directory, 'conf.py'), 'w') as f: f.write(default_conf_py_template.format_map(self.template_variables)) def generate_wrapping_rosdoc2_sphinx_project_into_directory( self, wrapped_sphinx_directory, - sphinx_project_directory, + conf_py_directory, python_src_directory, intersphinx_mapping_extensions, ): """Generate the rosdoc2 sphinx project configuration files.""" - # Generate a default index.rst + index_rst_path = os.path.join(wrapped_sphinx_directory, 'index.rst') package = self.build_context.package - logger.info('Using a default index.rst.jinja') - template_path = resources.files('rosdoc2.verbs.build.builders').joinpath('index.rst.jinja') - template_jinja = template_path.read_text() + if not os.path.isfile(index_rst_path): + # Generate a default index.rst + logger.info('Using a default index.rst.jinja') + template_path = resources.files( + 'rosdoc2.verbs.build.builders').joinpath('index.rst.jinja') + template_jinja = template_path.read_text() - index_rst = Template(template_jinja).render(self.template_variables) + index_rst = Template(template_jinja).render(self.template_variables) - with open(os.path.join(wrapped_sphinx_directory, 'index.rst'), 'w+') as f: - f.write(index_rst) + with open(os.path.join(wrapped_sphinx_directory, 'index.rst'), 'w+') as f: + f.write(index_rst) - # Copy all user content, like images or documentation files, and - # source files to the wrapping directory - # - # If the user created an index.rst, it will overwrite our default here. Later we will - # overwrite any user's conf.py with a wrapped version, that also includes any user's - # conf.py variables. - if sphinx_project_directory: - try: - shutil.copytree( - os.path.abspath(sphinx_project_directory), - os.path.abspath(wrapped_sphinx_directory), - dirs_exist_ok=True) - - except OSError as e: - print(f'Failed to copy user content: {e}') - - package = self.build_context.package breathe_projects = [] if self.doxygen_xml_directory is not None: breathe_projects.append( @@ -705,7 +693,7 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory( 'did_run_doxygen': self.doxygen_xml_directory is not None, 'wrapped_sphinx_directory': esc_backslash(os.path.abspath(wrapped_sphinx_directory)), 'user_conf_py_filename': esc_backslash( - os.path.abspath(os.path.join(sphinx_project_directory, 'conf.py'))), + os.path.abspath(os.path.join(conf_py_directory, 'conf.py'))), 'breathe_projects': ',\n'.join(breathe_projects) + '\n ', 'intersphinx_mapping_extensions': ',\n '.join(intersphinx_mapping_extensions), 'package': package, @@ -715,5 +703,5 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory( 'package_version_short': '.'.join(package.version.split('.')[0:2]), }) - with open(os.path.join(wrapped_sphinx_directory, 'conf.py'), 'w+') as f: + with open(os.path.join(wrapped_sphinx_directory, 'conf.py'), 'w') as f: f.write(rosdoc2_wrapping_conf_py_template.format_map(self.template_variables)) diff --git a/rosdoc2/verbs/build/include_user_docs.py b/rosdoc2/verbs/build/include_user_docs.py index a7e57902..ca9fe201 100644 --- a/rosdoc2/verbs/build/include_user_docs.py +++ b/rosdoc2/verbs/build/include_user_docs.py @@ -44,39 +44,39 @@ """ -def include_user_docs(package_dir: str, +def include_user_docs(sphinx_project_directory: str, output_dir: str, ): """Generate rst files for user documents.""" - logger.info(f'include_user_docs: package_dir {package_dir} output_dir {output_dir}') - # Search the ./doc directory - doc_dir = os.path.join(package_dir, 'doc') - # Search /doc to insure there is at least one item of renderable documentation + logger.info(f'include_user_docs: sphinx_project_directory {sphinx_project_directory} ' + f'output_dir {output_dir}') doc_directories = [] - for root, _, files in os.walk(doc_dir): + for root, _, files in os.walk(sphinx_project_directory): for file in files: # ensure a valid documentation file exists, directories might only contain resources. (_, ext) = os.path.splitext(file) if ext in ['.rst', '.md', '.markdown']: logger.debug(f'Found renderable documentation file in {root} named {file}') - relpath = os.path.relpath(root, doc_dir) + relpath = os.path.relpath(root, sphinx_project_directory) relpath = relpath.replace('\\', '/') doc_directories.append(relpath) break if not doc_directories: - logger.debug('no documentation found in /doc') + logger.debug(f'no documentation found in {sphinx_project_directory}') return doc_directories - logger.info(f'Documentation found in /doc in directories {doc_directories}') + logger.info(f'Documentation found in directories {doc_directories}') # At this point we know that there are some directories that have documentation in them under # /doc, but we do not know which ones might also be needed for images or includes. So we copy # everything to the output directory. shutil.copytree( - os.path.abspath(doc_dir), + os.path.abspath(sphinx_project_directory), os.path.abspath(os.path.join(output_dir, 'user_docs')), dirs_exist_ok=True) + logger.info(f'Copying {os.path.abspath(sphinx_project_directory)} to ' + f'{os.path.abspath(os.path.join(output_dir, "user_docs"))}') toc_content = documentation_rst_template # generate a glob rst entry for each directory with documents for relpath in doc_directories: diff --git a/test/packages/false_python/docs/somedocs.md b/test/packages/false_python/docs/somedocs.md new file mode 100644 index 00000000..8deb6111 --- /dev/null +++ b/test/packages/false_python/docs/somedocs.md @@ -0,0 +1,3 @@ +# This is documentation + +blah, blah diff --git a/test/packages/false_python/package.xml b/test/packages/false_python/package.xml index ef8fcbb8..8602d5b2 100644 --- a/test/packages/false_python/package.xml +++ b/test/packages/false_python/package.xml @@ -8,5 +8,6 @@ Apache-2.0 ament_python + rosdoc2.yaml diff --git a/test/packages/false_python/rosdoc2.yaml b/test/packages/false_python/rosdoc2.yaml new file mode 100644 index 00000000..997a954e --- /dev/null +++ b/test/packages/false_python/rosdoc2.yaml @@ -0,0 +1,59 @@ +## Default configuration, generated by rosdoc2. + +## This 'attic section' self-documents this file's type and version. +type: 'rosdoc2 config' +version: 1 + +--- + +settings: { + ## This setting is relevant mostly if the standard Python package layout cannot + ## be assumed for 'sphinx-apidoc' invocation. The user can provide the path + ## (relative to the 'package.xml' file) where the Python modules defined by this + ## package are located. + # python_source: 'false_python', + + ## This setting, if true, attempts to run `doxygen` and the `breathe`/`exhale` + ## extensions to `sphinx` regardless of build type. This is most useful if the + ## user would like to generate C/C++ API documentation for a package that is not + ## of the `ament_cmake/cmake` build type. + # always_run_doxygen: false, + + ## This setting, if true, attempts to run `sphinx-apidoc` regardless of build + ## type. This is most useful if the user would like to generate Python API + ## documentation for a package that is not of the `ament_python` build type. + # always_run_sphinx_apidoc: false, + + # This setting, if provided, will override the build_type of this package + # for documentation purposes only. If not provided, documentation will be + # generated assuming the build_type in package.xml. + # override_build_type: 'ament_python', +} +builders: + ## Each stanza represents a separate build step, performed by a specific 'builder'. + ## The key of each stanza is the builder to use; this must be one of the + ## available builders. + ## The value of each stanza is a dictionary of settings for the builder that + ## outputs to that directory. + ## Keys in all settings dictionary are: + ## * 'output_dir' - determines output subdirectory for builder instance + ## relative to --output-directory + ## * 'name' - used when referencing the built docs from the index. + + - doxygen: { + # name: 'false_python Public C/C++ API', + # output_dir: 'generated/doxygen', + ## file name for a user-supplied Doxyfile + # doxyfile: null, + ## additional statements to add to the Doxyfile, list of strings + # extra_doxyfile_statements: [], + } + - sphinx: { + # name: 'false_python', + ## This path is relative to output staging. + # doxygen_xml_directory: 'generated/doxygen/xml', + # output_dir: '', + ## Root folder for the user-supplied documentation. If not specified, either 'doc' or + ## 'doc/source' will be used if renderable documentation is found there. + sphinx_sourcedir: 'docs', + } diff --git a/test/test_builder.py b/test/test_builder.py index ee75ea89..d13c65dd 100644 --- a/test/test_builder.py +++ b/test/test_builder.py @@ -132,11 +132,16 @@ def test_false_python(module_dir): do_build_package(DATAPATH / PKG_NAME, module_dir) excludes = ['python api'] - includes = ['I say I am python, but no actual python'] + includes = [ + 'I say I am python, but no actual python', + 'this is documentation' # the title of included documentation + ] + links_exist = ['user_docs.html'] # Found docs in a non-standard location do_test_package(PKG_NAME, module_dir, includes=includes, - excludes=excludes) + excludes=excludes, + links_exist=links_exist) def test_invalid_python_source(module_dir): From 681bc07b58b965734a5ee9bb51e7e0bccc7b9f2d Mon Sep 17 00:00:00 2001 From: "R. Kent James" Date: Mon, 5 Aug 2024 16:46:02 -0700 Subject: [PATCH 2/3] Use real directory names in wrapped directory --- .../verbs/build/builders/sphinx_builder.py | 32 +++++++++++-------- rosdoc2/verbs/build/include_user_docs.py | 32 +++++++++++-------- .../src_python/doc/source/thedocs.rst | 4 +++ test/test_builder.py | 5 ++- test/utils.py | 2 +- 5 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 test/packages/src_python/doc/source/thedocs.rst diff --git a/rosdoc2/verbs/build/builders/sphinx_builder.py b/rosdoc2/verbs/build/builders/sphinx_builder.py index 5382c90d..89befbe1 100644 --- a/rosdoc2/verbs/build/builders/sphinx_builder.py +++ b/rosdoc2/verbs/build/builders/sphinx_builder.py @@ -392,10 +392,10 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context): if key in ['name', 'output_dir', 'doxygen_xml_directory']: continue if key == 'sphinx_sourcedir': - if not value: + if value is None: continue - sphinx_sourcedir = os.path.join(configuration_file_dir, value) - if not os.path.isdir(sphinx_sourcedir): + sphinx_sourcedir = value + if not os.path.isdir(os.path.join(configuration_file_dir, value)): raise RuntimeError( f"Error Sphinx SOURCEDIR '{value}' does not exist relative " f"to '{configuration_file_path}', or is not a directory.") @@ -494,10 +494,14 @@ def build(self, *, doc_build_folder, output_staging_directory): # include user documentation if sphinx_project_directory: - doc_directories = include_user_docs(sphinx_project_directory, wrapped_sphinx_directory) + doc_directories = include_user_docs( + sphinx_project_directory, wrapped_sphinx_directory, package_xml_directory) logger.info(f'doc_directories: {doc_directories}') else: doc_directories = None + sphinx_project_directory = 'doc' + os.makedirs(os.path.join(wrapped_sphinx_directory, sphinx_project_directory), + exist_ok=True) # Collect intersphinx mapping extensions from discovered inventory files. inventory_files = \ @@ -529,12 +533,14 @@ def build(self, *, doc_build_folder, output_staging_directory): # If the user did no include a conf.py in sphinx_project_directory, generate # a default conf.py - conf_py_directory = sphinx_project_directory - if not conf_py_directory or not os.path.isfile(os.path.join(conf_py_directory, 'conf.py')): + conf_py_directory = os.path.join(wrapped_sphinx_directory, sphinx_project_directory) + logger.info(f'conf_py_directory: {conf_py_directory}') + if os.path.isfile(os.path.join(conf_py_directory, 'conf.py')): + os.rename(os.path.join(conf_py_directory, 'conf.py'), + os.path.join(conf_py_directory, '__conf.py')) + else: logger.info('Note: no conf.py provided by the user, ' 'therefore using a default Sphinx configuration.') - conf_py_directory = os.path.join(doc_build_folder, 'default_sphinx_project') - os.makedirs(conf_py_directory, exist_ok=True) self.generate_default_project_into_directory( conf_py_directory, python_src_directory) @@ -632,11 +638,11 @@ def locate_sphinx_sourcedir_from_standard_locations(self): """ package_xml_directory = os.path.dirname(self.build_context.package.filename) options = [ - os.path.join(package_xml_directory, 'doc'), - os.path.join(package_xml_directory, 'doc', 'source'), + os.path.join('doc', 'source'), + 'doc', ] for option in options: - if os.path.isdir(option): + if os.path.isdir(os.path.join(package_xml_directory, option)): return option return None @@ -654,7 +660,7 @@ def generate_default_project_into_directory( )), }) - with open(os.path.join(conf_py_directory, 'conf.py'), 'w') as f: + with open(os.path.join(conf_py_directory, '__conf.py'), 'w') as f: f.write(default_conf_py_template.format_map(self.template_variables)) def generate_wrapping_rosdoc2_sphinx_project_into_directory( @@ -693,7 +699,7 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory( 'did_run_doxygen': self.doxygen_xml_directory is not None, 'wrapped_sphinx_directory': esc_backslash(os.path.abspath(wrapped_sphinx_directory)), 'user_conf_py_filename': esc_backslash( - os.path.abspath(os.path.join(conf_py_directory, 'conf.py'))), + os.path.abspath(os.path.join(conf_py_directory, '__conf.py'))), 'breathe_projects': ',\n'.join(breathe_projects) + '\n ', 'intersphinx_mapping_extensions': ',\n '.join(intersphinx_mapping_extensions), 'package': package, diff --git a/rosdoc2/verbs/build/include_user_docs.py b/rosdoc2/verbs/build/include_user_docs.py index ca9fe201..db626edf 100644 --- a/rosdoc2/verbs/build/include_user_docs.py +++ b/rosdoc2/verbs/build/include_user_docs.py @@ -28,7 +28,7 @@ :maxdepth: 1 :glob: - user_docs/* + {sphinx_project_directory}/* """ subdirectory_rst_template = """\ @@ -40,26 +40,27 @@ :maxdepth: 2 :glob: - user_docs/{name}/* + {sphinx_project_directory}/{name}/* """ def include_user_docs(sphinx_project_directory: str, output_dir: str, + package_xml_directory: str ): """Generate rst files for user documents.""" - logger.info(f'include_user_docs: sphinx_project_directory {sphinx_project_directory} ' - f'output_dir {output_dir}') + logger.info(f'include_user_docs: sphinx_project_directory <{sphinx_project_directory}> ' + f'output_dir <{output_dir}>') + user_doc_directory = os.path.join( + os.path.join(package_xml_directory, sphinx_project_directory)) doc_directories = [] - for root, _, files in os.walk(sphinx_project_directory): + for root, _, files in os.walk(user_doc_directory): for file in files: # ensure a valid documentation file exists, directories might only contain resources. (_, ext) = os.path.splitext(file) if ext in ['.rst', '.md', '.markdown']: logger.debug(f'Found renderable documentation file in {root} named {file}') - relpath = os.path.relpath(root, sphinx_project_directory) - relpath = relpath.replace('\\', '/') - doc_directories.append(relpath) + doc_directories.append(os.path.relpath(root, user_doc_directory)) break if not doc_directories: @@ -70,14 +71,15 @@ def include_user_docs(sphinx_project_directory: str, # At this point we know that there are some directories that have documentation in them under # /doc, but we do not know which ones might also be needed for images or includes. So we copy # everything to the output directory. + logger.info(f'Copying {os.path.join(package_xml_directory, sphinx_project_directory)} to ' + f'{os.path.join(output_dir, sphinx_project_directory)}') shutil.copytree( - os.path.abspath(sphinx_project_directory), - os.path.abspath(os.path.join(output_dir, 'user_docs')), + os.path.join(package_xml_directory, sphinx_project_directory), + os.path.join(output_dir, sphinx_project_directory), dirs_exist_ok=True) - logger.info(f'Copying {os.path.abspath(sphinx_project_directory)} to ' - f'{os.path.abspath(os.path.join(output_dir, "user_docs"))}') - toc_content = documentation_rst_template + toc_content = documentation_rst_template.format_map( + {'sphinx_project_directory': sphinx_project_directory}) # generate a glob rst entry for each directory with documents for relpath in doc_directories: # directories that will be explicitly listed in index.rst @@ -85,7 +87,9 @@ def include_user_docs(sphinx_project_directory: str, continue docname = 'user_docs_' + slugify(relpath) # This is the name that sphinx uses content = subdirectory_rst_template.format_map( - {'name': relpath, 'name_underline': '=' * len(relpath)}) + {'name': relpath, + 'name_underline': '=' * len(relpath), + 'sphinx_project_directory': sphinx_project_directory}) sub_path = os.path.join(output_dir, docname + '.rst') with open(sub_path, 'w+') as f: f.write(content) diff --git a/test/packages/src_python/doc/source/thedocs.rst b/test/packages/src_python/doc/source/thedocs.rst new file mode 100644 index 00000000..c1d0e958 --- /dev/null +++ b/test/packages/src_python/doc/source/thedocs.rst @@ -0,0 +1,4 @@ +Documentation in source +======================= + +This documentation is in the doc/source location instead of doc/ diff --git a/test/test_builder.py b/test/test_builder.py index d13c65dd..25424d1f 100644 --- a/test/test_builder.py +++ b/test/test_builder.py @@ -117,7 +117,10 @@ def test_src_python(module_dir): PKG_NAME = 'src_python' do_build_package(DATAPATH / PKG_NAME, module_dir) - includes = ['src_python package'] + includes = [ + 'src_python package', + 'documentation in source', # We found the documentation in doc/source + ] links_exist = ['src_python.html'] do_build_package(DATAPATH / PKG_NAME, module_dir) diff --git a/test/utils.py b/test/utils.py index 910e35d4..b122b1da 100644 --- a/test/utils.py +++ b/test/utils.py @@ -162,7 +162,7 @@ def do_test_full_package(module_dir, output_path='output', pkg_name='full_packag links_exist = [ f'{pkg_name}.dummy.html', 'modules.html', - 'user_docs/morestuff/more_of_more/subsub.html', # a deep documentation file + 'doc/morestuff/more_of_more/subsub.html', # a deep documentation file 'standards.html', 'https://example.com/repo', 'standard_docs/PACKAGE.html', # package.xml From 145416ed4045d53d61035a6deb8dda0a207cac39 Mon Sep 17 00:00:00 2001 From: "R. Kent James" Date: Thu, 8 Aug 2024 14:20:15 -0700 Subject: [PATCH 3/3] Support user_doc_dir --- .../verbs/build/builders/sphinx_builder.py | 86 ++++++++++++------- .../build/inspect_package_for_settings.py | 6 ++ test/packages/default_yaml/rosdoc2.yaml | 6 ++ test/packages/empty_doc_dir/doc/dummy.stuff | 1 + test/packages/empty_doc_dir/package.xml | 9 ++ .../false_python/docs/moredocs/more1.rst | 4 + test/packages/false_python/rosdoc2.yaml | 21 +++-- .../has_sphinx_sourcedir/doc/index.rst | 13 +++ .../doc/moredocs/more1.rst | 4 + .../has_sphinx_sourcedir/doc/somedocs.md | 3 + .../packages/has_sphinx_sourcedir/package.xml | 13 +++ .../has_sphinx_sourcedir/rosdoc2.yaml | 67 +++++++++++++++ test/test_builder.py | 49 ++++++++++- 13 files changed, 243 insertions(+), 39 deletions(-) create mode 100644 test/packages/empty_doc_dir/doc/dummy.stuff create mode 100644 test/packages/empty_doc_dir/package.xml create mode 100644 test/packages/false_python/docs/moredocs/more1.rst create mode 100644 test/packages/has_sphinx_sourcedir/doc/index.rst create mode 100644 test/packages/has_sphinx_sourcedir/doc/moredocs/more1.rst create mode 100644 test/packages/has_sphinx_sourcedir/doc/somedocs.md create mode 100644 test/packages/has_sphinx_sourcedir/package.xml create mode 100644 test/packages/has_sphinx_sourcedir/rosdoc2.yaml diff --git a/rosdoc2/verbs/build/builders/sphinx_builder.py b/rosdoc2/verbs/build/builders/sphinx_builder.py index 89befbe1..8d8b62d3 100644 --- a/rosdoc2/verbs/build/builders/sphinx_builder.py +++ b/rosdoc2/verbs/build/builders/sphinx_builder.py @@ -362,6 +362,12 @@ class SphinxBuilder(Builder): - If set, the documentation in this folder replaces the default documentation generated by rosdoc2. That is, the directory you would pass to sphinx-build as SOURCEDIR. Defaults to empty (or null in yaml, None in python). + - user_doc_dir (str) (optional) + - Directory (relative to the package.xml directory) where user documentation is found. If + documentation is in one of the standard locations (doc/ or doc/source) this is not + needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2 + output, but includes this user documentation along with other items included by default. + """ def __init__(self, builder_name, builder_entry_dictionary, build_context): @@ -376,6 +382,7 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context): self.name = self.name or self.build_context.package.name self.output_dir = self.output_dir or '' self.sphinx_sourcedir = None + self.user_doc_dir = None # Must check for the existence of this later, as it may not have been made yet. self.doxygen_xml_directory = \ @@ -394,14 +401,19 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context): if key == 'sphinx_sourcedir': if value is None: continue - sphinx_sourcedir = value if not os.path.isdir(os.path.join(configuration_file_dir, value)): raise RuntimeError( f"Error Sphinx SOURCEDIR '{value}' does not exist relative " f"to '{configuration_file_path}', or is not a directory.") - else: - logger.info(f'The user specified sphinx_sourcedir as {sphinx_sourcedir}') - self.sphinx_sourcedir = sphinx_sourcedir + logger.info(f'The user specified sphinx_sourcedir as {value}') + self.sphinx_sourcedir = value + elif key == 'user_doc_dir': + if not os.path.isdir(os.path.join(configuration_file_dir, value)): + raise RuntimeError( + f"Error user documentation directory '{value}' does not exist relative " + f"to '{configuration_file_path}', or is not a directory.") + logger.info(f'The user specified user_doc_dir as {value}') + self.user_doc_dir = value else: raise RuntimeError(f"Error the Sphinx builder does not support key '{key}'") @@ -478,30 +490,36 @@ def build(self, *, doc_build_folder, output_staging_directory): logger.info(f'standard_docs: {standard_docs}') # Check if the user provided a sphinx directory. - sphinx_project_directory = self.sphinx_sourcedir - if sphinx_project_directory is not None: + doc_directories = [] + user_doc_dir = None + if self.sphinx_sourcedir is not None: # We do not need to check if this directory exists, as that was done in __init__. logger.info( 'Note: the user provided sourcedir for Sphinx ' - f"'{sphinx_project_directory}' will be used.") + f"'{self.sphinx_sourcedir}' will be used.") + # Copy all user content, like images or documentation files, and + # source files to the wrapping directory + try: + shutil.copytree( + os.path.join(package_xml_directory, self.sphinx_sourcedir), + wrapped_sphinx_directory, dirs_exist_ok=True) + except OSError as e: + print(f'Failed to copy user content: {e}') else: - # If the user does not supply a Sphinx sourcedir, check the standard locations. - sphinx_project_directory = self.locate_sphinx_sourcedir_from_standard_locations() - if sphinx_project_directory is not None: - logger.info( - 'Note: no sourcedir provided, but a Sphinx sourcedir located in the ' - f"standard location '{sphinx_project_directory}' and that will be used.") - - # include user documentation - if sphinx_project_directory: - doc_directories = include_user_docs( - sphinx_project_directory, wrapped_sphinx_directory, package_xml_directory) - logger.info(f'doc_directories: {doc_directories}') - else: - doc_directories = None - sphinx_project_directory = 'doc' - os.makedirs(os.path.join(wrapped_sphinx_directory, sphinx_project_directory), - exist_ok=True) + # include user documentation + if self.user_doc_dir: + user_doc_dir = self.user_doc_dir + else: + # If the user does not supply a doc directory, check the standard locations. + user_doc_dir = self.locate_user_doc_dir_from_standard_locations() + if user_doc_dir: + logger.info( + 'Note: no user_doc_dir provided, but a doc directory was located in a ' + f'standard location "{user_doc_dir}" and that will be used.') + if user_doc_dir: + doc_directories = include_user_docs( + user_doc_dir, wrapped_sphinx_directory, package_xml_directory) + logger.info(f'doc_directories: {doc_directories}') # Collect intersphinx mapping extensions from discovered inventory files. inventory_files = \ @@ -531,16 +549,20 @@ def build(self, *, doc_build_folder, output_staging_directory): 'package': self.build_context.package, }) - # If the user did no include a conf.py in sphinx_project_directory, generate - # a default conf.py - conf_py_directory = os.path.join(wrapped_sphinx_directory, sphinx_project_directory) - logger.info(f'conf_py_directory: {conf_py_directory}') - if os.path.isfile(os.path.join(conf_py_directory, 'conf.py')): + # If the user did no include a conf.py, generate a default conf.py + if self.sphinx_sourcedir: + conf_py_directory = wrapped_sphinx_directory + elif user_doc_dir: + conf_py_directory = os.path.join(wrapped_sphinx_directory, user_doc_dir) + else: + conf_py_directory = None + if conf_py_directory and os.path.isfile(os.path.join(conf_py_directory, 'conf.py')): os.rename(os.path.join(conf_py_directory, 'conf.py'), os.path.join(conf_py_directory, '__conf.py')) else: logger.info('Note: no conf.py provided by the user, ' 'therefore using a default Sphinx configuration.') + conf_py_directory = wrapped_sphinx_directory self.generate_default_project_into_directory( conf_py_directory, python_src_directory) @@ -625,11 +647,11 @@ def build(self, *, doc_build_folder, output_staging_directory): # Return the directory into which Sphinx generated. return sphinx_output_dir - def locate_sphinx_sourcedir_from_standard_locations(self): + def locate_user_doc_dir_from_standard_locations(self): """ - Return the location of a Sphinx project for the package. + Return the location of a user documentation for the package. - If the sphinx configuration exists in a standard location, return it, + If the user documentation exists in a standard location, return it, otherwise return None. The standard locations are '/doc/source' and '/doc', for projects that selected diff --git a/rosdoc2/verbs/build/inspect_package_for_settings.py b/rosdoc2/verbs/build/inspect_package_for_settings.py index f7ed8cf7..c95fb3c8 100644 --- a/rosdoc2/verbs/build/inspect_package_for_settings.py +++ b/rosdoc2/verbs/build/inspect_package_for_settings.py @@ -80,6 +80,12 @@ ## doc/source/ folder will still be included by default, along with other relevant package ## information. # sphinx_sourcedir: null, + ## Directory (relative to the package.xml directory) where user documentation is found. If + ## documentation is in one of the standard locations (doc/ or doc/source) this is not + ## needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2 + ## output, but includes this user documentation along with other items included by default + ## by rosdoc2. + # user_doc_dir: 'doc' }} """ diff --git a/test/packages/default_yaml/rosdoc2.yaml b/test/packages/default_yaml/rosdoc2.yaml index 954bddba..2dec477a 100644 --- a/test/packages/default_yaml/rosdoc2.yaml +++ b/test/packages/default_yaml/rosdoc2.yaml @@ -59,4 +59,10 @@ builders: ## doc/source/ folder will still be included by default, along with other relevant package ## information. sphinx_sourcedir: null, + ## Directory (relative to the package.xml directory) where user documentation is found. If + ## documentation is in one of the standard locations (doc/ or doc/source) this is not + ## needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2 + ## output, but includes this user documentation along with other items included by default + ## by rosdoc2. + user_doc_dir: 'doc' } diff --git a/test/packages/empty_doc_dir/doc/dummy.stuff b/test/packages/empty_doc_dir/doc/dummy.stuff new file mode 100644 index 00000000..6aa232f4 --- /dev/null +++ b/test/packages/empty_doc_dir/doc/dummy.stuff @@ -0,0 +1 @@ +This file does not contain any documentation. diff --git a/test/packages/empty_doc_dir/package.xml b/test/packages/empty_doc_dir/package.xml new file mode 100644 index 00000000..25eec54f --- /dev/null +++ b/test/packages/empty_doc_dir/package.xml @@ -0,0 +1,9 @@ + + + + empty_doc_dir + 0.1.2 + Package with an empty doc directory + Some One + Apache License 2.0 + diff --git a/test/packages/false_python/docs/moredocs/more1.rst b/test/packages/false_python/docs/moredocs/more1.rst new file mode 100644 index 00000000..e187fab7 --- /dev/null +++ b/test/packages/false_python/docs/moredocs/more1.rst @@ -0,0 +1,4 @@ +This is some more documentation +=============================== + +blah, blah. diff --git a/test/packages/false_python/rosdoc2.yaml b/test/packages/false_python/rosdoc2.yaml index 997a954e..a795775a 100644 --- a/test/packages/false_python/rosdoc2.yaml +++ b/test/packages/false_python/rosdoc2.yaml @@ -24,9 +24,9 @@ settings: { ## documentation for a package that is not of the `ament_python` build type. # always_run_sphinx_apidoc: false, - # This setting, if provided, will override the build_type of this package - # for documentation purposes only. If not provided, documentation will be - # generated assuming the build_type in package.xml. + ## This setting, if provided, will override the build_type of this package + ## for documentation purposes only. If not provided, documentation will be + ## generated assuming the build_type in package.xml. # override_build_type: 'ament_python', } builders: @@ -53,7 +53,16 @@ builders: ## This path is relative to output staging. # doxygen_xml_directory: 'generated/doxygen/xml', # output_dir: '', - ## Root folder for the user-supplied documentation. If not specified, either 'doc' or - ## 'doc/source' will be used if renderable documentation is found there. - sphinx_sourcedir: 'docs', + ## If sphinx_sourcedir is specified and not null, then the documentation in that folder + ## (specified relative to the package.xml directory) will replace rosdoc2's normal output. + ## If sphinx_sourcedir is left unspecified, any documentation found in the doc/ or + ## doc/source/ folder will still be included by default, along with other relevant package + ## information. + # sphinx_sourcedir: null, + ## Directory (relative to the package.xml directory) where user documentation is found. If + ## documentation is in one of the standard locations (doc/ or doc/source) this is not + ## needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2 + ## output, but includes this user documentation along with other items included by default + ## by rosdoc2. + user_doc_dir: 'docs' } diff --git a/test/packages/has_sphinx_sourcedir/doc/index.rst b/test/packages/has_sphinx_sourcedir/doc/index.rst new file mode 100644 index 00000000..fae3a2cb --- /dev/null +++ b/test/packages/has_sphinx_sourcedir/doc/index.rst @@ -0,0 +1,13 @@ +I defined sphinx_sourcedir +========================== + +This content will replace all other content! + +Additional Documentation +------------------------ + +.. toctree:: + :titlesonly: + :glob: + + moredocs/* diff --git a/test/packages/has_sphinx_sourcedir/doc/moredocs/more1.rst b/test/packages/has_sphinx_sourcedir/doc/moredocs/more1.rst new file mode 100644 index 00000000..e187fab7 --- /dev/null +++ b/test/packages/has_sphinx_sourcedir/doc/moredocs/more1.rst @@ -0,0 +1,4 @@ +This is some more documentation +=============================== + +blah, blah. diff --git a/test/packages/has_sphinx_sourcedir/doc/somedocs.md b/test/packages/has_sphinx_sourcedir/doc/somedocs.md new file mode 100644 index 00000000..8deb6111 --- /dev/null +++ b/test/packages/has_sphinx_sourcedir/doc/somedocs.md @@ -0,0 +1,3 @@ +# This is documentation + +blah, blah diff --git a/test/packages/has_sphinx_sourcedir/package.xml b/test/packages/has_sphinx_sourcedir/package.xml new file mode 100644 index 00000000..c3a55851 --- /dev/null +++ b/test/packages/has_sphinx_sourcedir/package.xml @@ -0,0 +1,13 @@ + + + + has_sphinx_sourcedir + 0.0.0 + I specify sphinx_sourcedir to override standard output + Ye ol' Python Pro + Apache-2.0 + + ament_python + rosdoc2.yaml + + diff --git a/test/packages/has_sphinx_sourcedir/rosdoc2.yaml b/test/packages/has_sphinx_sourcedir/rosdoc2.yaml new file mode 100644 index 00000000..eb6fc2ca --- /dev/null +++ b/test/packages/has_sphinx_sourcedir/rosdoc2.yaml @@ -0,0 +1,67 @@ +## Default configuration, generated by rosdoc2. + +## This 'attic section' self-documents this file's type and version. +type: 'rosdoc2 config' +version: 1 + +--- + +settings: { + ## This setting is relevant mostly if the standard Python package layout cannot + ## be assumed for 'sphinx-apidoc' invocation. The user can provide the path + ## (relative to the 'package.xml' file) where the Python modules defined by this + ## package are located. + # python_source: 'false_python', + + ## This setting, if true, attempts to run `doxygen` and the `breathe`/`exhale` + ## extensions to `sphinx` regardless of build type. This is most useful if the + ## user would like to generate C/C++ API documentation for a package that is not + ## of the `ament_cmake/cmake` build type. + # always_run_doxygen: false, + + ## This setting, if true, attempts to run `sphinx-apidoc` regardless of build + ## type. This is most useful if the user would like to generate Python API + ## documentation for a package that is not of the `ament_python` build type. + # always_run_sphinx_apidoc: false, + + ## This setting, if provided, will override the build_type of this package + ## for documentation purposes only. If not provided, documentation will be + ## generated assuming the build_type in package.xml. + # override_build_type: 'ament_python', +} +builders: + ## Each stanza represents a separate build step, performed by a specific 'builder'. + ## The key of each stanza is the builder to use; this must be one of the + ## available builders. + ## The value of each stanza is a dictionary of settings for the builder that + ## outputs to that directory. + ## Keys in all settings dictionary are: + ## * 'output_dir' - determines output subdirectory for builder instance + ## relative to --output-directory + ## * 'name' - used when referencing the built docs from the index. + + - doxygen: { + # name: 'false_python Public C/C++ API', + # output_dir: 'generated/doxygen', + ## file name for a user-supplied Doxyfile + # doxyfile: null, + ## additional statements to add to the Doxyfile, list of strings + # extra_doxyfile_statements: [], + } + - sphinx: { + ## This path is relative to output staging. + # doxygen_xml_directory: 'generated/doxygen/xml', + # output_dir: '', + ## If sphinx_sourcedir is specified and not null, then the documentation in that folder + ## (specified relative to the package.xml directory) will replace rosdoc2's normal output. + ## If sphinx_sourcedir is left unspecified, any documentation found in the doc/ or + ## doc/source/ folder will still be included by default, along with other relevant package + ## information. + sphinx_sourcedir: 'doc', + ## Directory (relative to the package.xml directory) where user documentation is found. If + ## documentation is in one of the standard locations (doc/ or doc/source) this is not + ## needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2 + ## output, but includes this user documentation along with other items included by default + ## by rosdoc2. + # user_doc_dir: 'doc' + } diff --git a/test/test_builder.py b/test/test_builder.py index 25424d1f..5a5cdc24 100644 --- a/test/test_builder.py +++ b/test/test_builder.py @@ -139,7 +139,10 @@ def test_false_python(module_dir): 'I say I am python, but no actual python', 'this is documentation' # the title of included documentation ] - links_exist = ['user_docs.html'] # Found docs in a non-standard location + links_exist = [ + 'user_docs.html', # Found docs in a non-standard location + 'docs/moredocs/more1.html' # Found subdirectory in non-standard location + ] do_test_package(PKG_NAME, module_dir, includes=includes, @@ -204,3 +207,47 @@ def test_basic_cpp(module_dir): generated = pathlib.Path(DATAPATH / PKG_NAME / 'doc' / 'generated') assert not generated.exists(), \ 'Building should not create a "generated" directory in package/doc' + + +def test_has_sphinx_sourcedir(module_dir): + PKG_NAME = 'has_sphinx_sourcedir' + do_build_package(DATAPATH / PKG_NAME, module_dir) + + includes = [ + 'i defined sphinx_sourcedir' + ] + excludes = [ + 'standards.html', # We override normal rosdoc2, so this should be missing. + ] + links_exist = [ + 'moredocs/more1.html', # Documentation in a subdirectory + ] + + do_test_package(PKG_NAME, module_dir, + includes=includes, + excludes=excludes, + links_exist=links_exist) + + do_test_package(PKG_NAME, module_dir) + + +def test_empty_doc_dir(module_dir): + PKG_NAME = 'empty_doc_dir' + do_build_package(DATAPATH / PKG_NAME, module_dir) + + includes = [ + 'package with an empty doc directory', # The package description + ] + excludes = [ + 'documentation' # We should not show empty documentation + ] + links_exist = [ + 'standards.html', # We still show the package + ] + + do_test_package(PKG_NAME, module_dir, + includes=includes, + excludes=excludes, + links_exist=links_exist) + + do_test_package(PKG_NAME, module_dir)