Properly include generated Python files in the wheel (#12)

When building the wheel, the setuptools file discovery feature runs
before the `compile_betterproto` sub-command is run, so the files are
not included in the resulting wheel.

To overcome this issue we use a hack, by running the compile command
early when the distribution options are being finalized, we make sure
the files are present by the time the file discovery runs.

This is very hacky though, and even when it seems to work well in all
tested cases (calling setuptools` `setup()` directly, `build`, `pip
install`, `pip install -e`), I wouldn't be very surprised if it breaks
in the future, so we probably will have to have an eye on it.

Unfortunately it seems there is no easy way to do this with setuptools:

*
pypa/setuptools#3180 (comment)
* pypa/setuptools#2591

This PR also includes some other improvements, including:

* Better abstraction of the configuration object
* Better hooking into setuptools
* Use of logging instead of print
* Better name for the command to compile the proto files

README.md

@@ -64,14 +64,6 @@ You should add [betterproto] as a dependency too, for example:
 dependencies = ["betterproto == 2.0.0b6"]
 ```
 
-You probably also need to add the protobuf files to the `MANIFEST.in` file, so
-they are included in the source distribution. For example (following our
-customized example):
-
-```plaintext
-recursive-include proto *.proto
-```
-
 Once this is done, the conversion of the proto files to Python files should be
 automatic. Just try building the package with:
 
@@ -83,7 +75,7 @@ python -m build
 A new command to generate the files will be also added to `setuptools`, you can
 run it manually with:
 ```sh
-python -c 'import setuptools; setuptools.setup()' build_betterproto
+python -c 'import setuptools; setuptools.setup()' compile_betterproto
 ```
 
 You can also pass the configuration options via command line for quick testing,

RELEASE_NOTES.md

@@ -1,17 +1,5 @@
 # Betterproto Setuptools plugin Release Notes
 
-## Summary
-
-<!-- Here goes a general summary of what this release is about -->
-
-## Upgrading
-
-<!-- Here goes notes on how to upgrade from previous versions, including deprecations and what they should be replaced with -->
-
-## New Features
-
-<!-- Here goes the main new features and examples or instructions on how to use them -->
-
 ## Bug Fixes
 
-<!-- Here goes notable bug fixes that are worth a special mention or explanation -->
+- Fix a bug where the generated files were not included in the wheel distribution.

pyproject.toml

@@ -46,8 +46,12 @@ dynamic = ["version"]
 name = "Frequenz Energy-as-a-Service GmbH"
 email = "[email protected]"
 
+[project.entry-points."setuptools.finalize_distribution_options"]
+finalize_distribution_options_betterproto = "setuptools_betterproto:finalize_distribution_options"
+
 [project.entry-points."distutils.commands"]
-build_betterproto = "setuptools_betterproto:CompileProto"
+compile_betterproto = "setuptools_betterproto:CompileBetterproto"
+add_proto_files = "setuptools_betterproto:AddProtoFiles"
 
 [project.optional-dependencies]
 dev-flake8 = [

src/setuptools_betterproto/__init__.py

@@ -3,10 +3,13 @@
 
 """A modern setuptools plugin to generate Python files from proto files using betterproto."""
 
-from ._command import CompileProto
+from ._command import AddProtoFiles, CompileBetterproto
 from ._config import ProtobufConfig
+from ._install import finalize_distribution_options
 
 __all__ = [
-    "CompileProto",
+    "AddProtoFiles",
+    "CompileBetterproto",
     "ProtobufConfig",
+    "finalize_distribution_options",
 ]

src/setuptools_betterproto/_command.py

@@ -1,28 +1,33 @@
 # License: MIT
 # Copyright © 2024 Frequenz Energy-as-a-Service GmbH
 
-"""Setuptool hooks to build protobuf files.
+"""Setuptools commands to compile and distribute protobuf files.
 
 This module contains a setuptools command that can be used to compile protocol
-buffer files in a project.
+buffer files in a project and to add the protobuf files to the source distribution.
 
-It also runs the command as the first sub-command for the build command, so
-protocol buffer files are compiled automatically before the project is built.
+To be able to ship the proto files with the source distribution, a command to replace
+the `sdist` command is also provided. This command will copy the proto files to the
+source distribution before building it.
 """
 
-import pathlib
+import logging
+import os
+import shutil
 import subprocess
 import sys
-from typing import Iterator
 
 import setuptools
-import setuptools.command.build as _build_command
+import setuptools.command.sdist
+from typing_extensions import override
 
 from . import _config
 
+_logger = logging.getLogger(__name__)
 
-class CompileProto(setuptools.Command):
-    """Build the Python protobuf files."""
+
+class BaseProtoCommand(setuptools.Command):
+    """A base class for commands that deal with protobuf files."""
 
     proto_path: str
     """The path of the root directory containing the protobuf files."""
@@ -36,6 +41,9 @@ class CompileProto(setuptools.Command):
     out_path: str
     """The path of the root directory where the Python files will be generated."""
 
+    config: _config.ProtobufConfig
+    """The configuration object for the command."""
+
     description: str = "compile protobuf files using betterproto"
     """Description of the command."""
 
@@ -59,53 +67,120 @@ class CompileProto(setuptools.Command):
     ]
     """Options of the command."""
 
+    @override
     def initialize_options(self) -> None:
-        """Initialize options."""
-        config = _config.ProtobufConfig.from_pyproject_toml()
+        """Initialize options with default values."""
+        self.config = _config.ProtobufConfig.from_pyproject_toml()
 
-        self.proto_path = config.proto_path
-        self.proto_glob = config.proto_glob
-        self.include_paths = ",".join(config.include_paths)
-        self.out_path = config.out_path
+        self.proto_path = self.config.proto_path
+        self.proto_glob = self.config.proto_glob
+        self.include_paths = ",".join(self.config.include_paths)
+        self.out_path = self.config.out_path
 
+    @override
     def finalize_options(self) -> None:
-        """Finalize options."""
+        """Finalize options by converting them to a ProtobufConfig object."""
+        self.config = _config.ProtobufConfig.from_strings(
+            proto_path=self.proto_path,
+            proto_glob=self.proto_glob,
+            include_paths=self.include_paths,
+            out_path=self.out_path,
+        )
+
 
-    def _expand_paths(
-        self, proto_path: pathlib.Path, proto_glob: str
-    ) -> Iterator[pathlib.Path]:
-        """Expand the paths to the proto files."""
-        return (p.relative_to(proto_path) for p in proto_path.rglob(proto_glob))
+class CompileBetterproto(BaseProtoCommand):
+    """A command to compile the protobuf files."""
 
+    @override
     def run(self) -> None:
-        """Compile the Python protobuf files."""
-        include_paths = list(map(pathlib.Path, self.include_paths.split(",")))
-        proto_path = pathlib.Path(self.proto_path)
-        proto_files = list(self._expand_paths(proto_path, self.proto_glob))
+        """Compile the protobuf files to Python."""
+        proto_files = self.config.expanded_proto_files
 
         if not proto_files:
-            print(
-                f"No proto files found in {self.proto_path}/**/{self.proto_glob}/, "
-                "skipping compilation of proto files."
+            _logger.warning(
+                "No proto files were found in the `proto_path` (%s) using `proto_glob` "
+                "(%s). You probably want to check if you `proto_path` and `proto_glob` "
+                "are configured correctly. We are not compiling any proto files!",
+                self.config.proto_path,
+                self.config.proto_glob,
             )
             return
 
         protoc_cmd = [
             sys.executable,
             "-m",
             "grpc_tools.protoc",
-            *(f"-I{p}" for p in [self.proto_path, *include_paths]),
-            f"--python_betterproto_out={self.out_path}",
-            *map(str, proto_files),
+            *(f"-I{p}" for p in [self.config.proto_path, *self.config.include_paths]),
+            f"--python_betterproto_out={self.config.out_path}",
+            *proto_files,
         ]
 
-        print(f"Compiling proto files via: {' '.join(protoc_cmd)}")
+        _logger.info("compiling proto files via: %s", " ".join(protoc_cmd))
         subprocess.run(protoc_cmd, check=True)
 
 
-# This adds the build_betterproto command to the build sub-command.
-# The name of the command is mapped to the class name in the pyproject.toml file,
-# in the [project.entry-points.distutils.commands] section.
-# The None value is an optional function that can be used to determine if the
-# sub-command should be executed or not.
-_build_command.build.sub_commands.insert(0, ("build_betterproto", None))
+class AddProtoFiles(BaseProtoCommand):
+    """A command to add the proto files to the source distribution."""
+
+    def run(self) -> None:
+        """Copy the proto files to the source distribution."""
+        proto_files = self.config.expanded_proto_files
+        include_files = self.config.expanded_include_files
+
+        if include_files and not proto_files:
+            _logger.warning(
+                "Some proto files (%s) were found in the `include_paths` (%s), but "
+                "no proto files were found in the `proto_path`. You probably want to "
+                "check if your `proto_path` (%s) and `proto_glob` (%s) are configured "
+                "correctly. We are not adding the found include files to the source "
+                "distribution automatically!",
+                len(include_files),
+                ", ".join(self.config.include_paths),
+                self.config.proto_path,
+                self.config.proto_glob,
+            )
+            return
+
+        if not proto_files:
+            _logger.warning(
+                "No proto files were found in the `proto_path` (%s) using `proto_glob` "
+                "(%s). You probably want to check if you `proto_path` and `proto_glob` "
+                "are configured correctly. We are not adding any proto files to the "
+                "source distribution automatically!",
+                self.config.proto_path,
+                self.config.proto_glob,
+            )
+            return
+
+        dest_dir = self.distribution.get_fullname()
+
+        for file in (*proto_files, *include_files):
+            self.copy_with_directories(file, os.path.join(dest_dir, file))
+
+        _logger.info("added %s proto files", len(proto_files) + len(include_files))
+
+    def copy_with_directories(self, src: str, dest: str) -> None:
+        """Copy a file from src to dest, creating the destination's directory tree.
+
+        Any directories that do not exist in dest will be created.
+
+        Args:
+            src: The full path of the source file.
+            dest: The full path of the destination file.
+        """
+        dest_dir = os.path.dirname(dest)
+        if not os.path.exists(dest_dir):
+            _logger.debug("creating directory %s", dest_dir)
+            os.makedirs(dest_dir)
+        _logger.info("adding proto file to %s", dest)
+        shutil.copyfile(src, dest)
+
+
+class SdistWithProtoFiles(setuptools.command.sdist.sdist):
+    """A command to build the source distribution with the proto files."""
+
+    @override
+    def run(self) -> None:
+        """Add the proto files to the source distribution before building it."""
+        self.run_command("add_proto_files")
+        super().run()

src/setuptools_betterproto/_config.py

@@ -5,6 +5,7 @@
 
 import dataclasses
 import logging
+import pathlib
 import tomllib
 from collections.abc import Sequence
 from typing import Any, Self
@@ -75,3 +76,42 @@ def from_pyproject_toml(
 
         attrs = dict(defaults, **{k: config[k] for k in (known_keys & config_keys)})
         return dataclasses.replace(default, **attrs)
+
+    @classmethod
+    def from_strings(
+        cls, *, proto_path: str, proto_glob: str, include_paths: str, out_path: str
+    ) -> Self:
+        """Create a new configuration from plain strings.
+
+        Args:
+            proto_path: The path of the root directory containing the protobuf files.
+            proto_glob: The glob pattern to use to find the protobuf files.
+            include_paths: The paths to add to the include path when compiling the
+                protobuf files.
+            out_path: The path of the root directory where the Python files will be
+                generated.
+
+        Returns:
+            The configuration.
+        """
+        return cls(
+            proto_path=proto_path,
+            proto_glob=proto_glob,
+            include_paths=[p.strip() for p in include_paths.split(",")],
+            out_path=out_path,
+        )
+
+    @property
+    def expanded_proto_files(self) -> list[str]:
+        """The files in the `proto_path` expanded according to the configured glob."""
+        proto_path = pathlib.Path(self.proto_path)
+        return [str(proto_file) for proto_file in proto_path.rglob(self.proto_glob)]
+
+    @property
+    def expanded_include_files(self) -> list[str]:
+        """The files in the `include_paths` expanded according to the configured glob."""
+        return [
+            str(proto_file)
+            for include_path in map(pathlib.Path, self.include_paths)
+            for proto_file in include_path.rglob(self.proto_glob)
+        ]