Add automatic framework reference docs (#2644)

Co-authored-by: Charles Beauville <[email protected]>

datasets/dev/build-flwr-datasets-docs.sh

@@ -28,3 +28,6 @@ fi
 # Note if a package cannot be reach via the recursive traversal, even if it has __all__, it won't be documented.
 echo "Generating the docs based on only the functionality given in the __all__."
 sphinx-build -M html source build
+
+# Remove the autogenerated source files after the build
+rm source/ref-api/*.rst

doc/build-versioned-docs.sh

@@ -76,6 +76,9 @@ END
 
     # Restore branch as it was to avoid conflicts
     git restore source/_templates
+    git restore source/_templates/autosummary || rm -rf source/_templates/autosummary
+    rm source/ref-api/*.rst
+
     if [ "$current_version" = "v1.5.0" ]; then
       git restore source/conf.py
     fi
@@ -100,6 +103,7 @@ for current_language in ${languages}; do
   export current_language
   sphinx-build -b html source/ build/html/${current_version}/${current_language} -A lang=True -D language=${current_language}
 done
+rm source/ref-api/*.rst
 
 # Copy main version to the root of the built docs
 cp -r build/html/main/en/* build/html/

doc/source/_templates/autosummary/base.rst

@@ -0,0 +1,5 @@
+{{ name | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+.. auto{{ objtype }}:: {{ objname }}

doc/source/_templates/autosummary/class.rst

@@ -0,0 +1,33 @@
+{{ name | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+   :members:
+   :show-inheritance:
+   :inherited-members:
+
+   {% block methods %}
+
+   {% if methods %}
+   .. rubric:: {{ _('Methods') }}
+
+   .. autosummary::
+   {% for item in methods %}
+      {% if item != "__init__" %}
+         ~{{ name }}.{{ item }}
+      {% endif %}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: {{ _('Attributes') }}
+
+   .. autosummary::
+   {% for item in attributes %}
+      ~{{ name }}.{{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}

doc/source/_templates/autosummary/module.rst

@@ -0,0 +1,66 @@
+{{ name | escape | underline}}
+
+.. automodule:: {{ fullname }}
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: Module Attributes
+
+   .. autosummary::
+      :toctree:
+   {% for item in attributes %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block functions %}
+   {% if functions %}
+   .. rubric:: {{ _('Functions') }}
+
+   .. autosummary::
+      :toctree:
+   {% for item in functions %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block classes %}
+   {% if classes %}
+   .. rubric:: {{ _('Classes') }}
+
+   .. autosummary::
+      :toctree:
+      :template: autosummary/class.rst
+   {% for item in classes %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block exceptions %}
+   {% if exceptions %}
+   .. rubric:: {{ _('Exceptions') }}
+
+   .. autosummary::
+      :toctree:
+   {% for item in exceptions %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+{% block modules %}
+{% if modules %}
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :template: autosummary/module.rst
+   :recursive:
+{% for item in modules %}
+   {{ item }}
+{%- endfor %}
+{% endif %}
+{% endblock %}

doc/source/conf.py

@@ -96,6 +96,7 @@
 extensions = [
     "sphinx.ext.napoleon",
     "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
     "sphinx.ext.mathjax",
     "sphinx.ext.viewcode",
     "sphinx.ext.graphviz",
@@ -109,6 +110,42 @@
     "nbsphinx",
 ]
 
+# Generate .rst files
+autosummary_generate = True
+
+# Document ONLY the objects from __all__ (present in __init__ files).
+# It will be done recursively starting from flwr.__init__
+# Starting point is controlled in the index.rst file.
+autosummary_ignore_module_all = False
+
+# Each class and function docs start with the path to it
+# Make the flwr_datasets.federated_dataset.FederatedDataset appear as FederatedDataset
+# The full name is still at the top of the page
+add_module_names = False
+
+def find_test_modules(package_path):
+    """Go through the python files and exclude every *_test.py file."""
+    full_path_modules = []
+    for root, dirs, files in os.walk(package_path):
+        for file in files:
+            if file.endswith('_test.py'):
+                # Construct the module path relative to the package directory
+                full_path = os.path.join(root, file)
+                relative_path = os.path.relpath(full_path, package_path)
+                # Convert file path to dotted module path
+                module_path = os.path.splitext(relative_path)[0].replace(os.sep, '.')
+                full_path_modules.append(module_path)
+    modules = []
+    for full_path_module in full_path_modules:
+        parts = full_path_module.split('.')
+        for i in range(len(parts)):
+            modules.append('.'.join(parts[i:]))
+    return modules
+
+# Stop from documenting the *_test.py files.
+# That's the only way to do that in autosummary (make the modules as mock_imports).
+autodoc_mock_imports = find_test_modules(os.path.abspath("../../src/py/flwr"))
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
@@ -174,7 +211,8 @@
     "evaluation": "explanation-federated-evaluation.html",
     "differential-privacy-wrappers": "explanation-differential-privacy.html",
     # Restructuring: references
-    "apiref-flwr": "ref-api-flwr.html",
+    "apiref-flwr": "ref-api/flwr.html",
+    "ref-api-flwr": "ref-api/flwr.html",
     "apiref-cli": "ref-api-cli.html",
     "examples": "ref-example-projects.html",
     "telemetry": "ref-telemetry.html",

doc/source/index.rst

@@ -119,11 +119,17 @@ References
 
 Information-oriented API reference and other reference material.
 
+.. autosummary::
+   :toctree: ref-api
+   :template: autosummary/module.rst
+   :caption: API reference
+   :recursive:
+
+      flwr
+
 .. toctree::
    :maxdepth: 2
-   :caption: API reference
 
-   ref-api-flwr
    ref-api-cli
 
 .. toctree::