From 300da0f69e7248add7cc8393872028b755945c29 Mon Sep 17 00:00:00 2001 From: Peter Chang Date: Fri, 15 Nov 2024 14:30:39 -0500 Subject: [PATCH 1/2] add protobuf serialization test --- python/packages/autogen-core/pyproject.toml | 6 +-- .../tests/protos/serialization_test.proto | 11 +++++ .../tests/protos/serialization_test_pb2.py | 29 ++++++++++++ .../tests/protos/serialization_test_pb2.pyi | 46 +++++++++++++++++++ .../protos/serialization_test_pb2_grpc.py | 4 ++ .../protos/serialization_test_pb2_grpc.pyi | 16 +++++++ .../autogen-core/tests/test_serialization.py | 31 +++++++++++++ 7 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 python/packages/autogen-core/tests/protos/serialization_test.proto create mode 100644 python/packages/autogen-core/tests/protos/serialization_test_pb2.py create mode 100644 python/packages/autogen-core/tests/protos/serialization_test_pb2.pyi create mode 100644 python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py create mode 100644 python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi diff --git a/python/packages/autogen-core/pyproject.toml b/python/packages/autogen-core/pyproject.toml index 79f9a099d9f..0dcfad01b48 100644 --- a/python/packages/autogen-core/pyproject.toml +++ b/python/packages/autogen-core/pyproject.toml @@ -75,7 +75,7 @@ dev-dependencies = [ [tool.ruff] extend = "../../pyproject.toml" -exclude = ["build", "dist", "src/autogen_core/application/protos"] +exclude = ["build", "dist", "src/autogen_core/application/protos", "tests/protos"] include = ["src/**", "samples/*.py", "docs/**/*.ipynb", "tests/**"] [tool.ruff.lint.per-file-ignores] @@ -85,7 +85,7 @@ include = ["src/**", "samples/*.py", "docs/**/*.ipynb", "tests/**"] [tool.pyright] extends = "../../pyproject.toml" include = ["src", "tests", "samples"] -exclude = ["src/autogen_core/application/protos"] +exclude = ["src/autogen_core/application/protos", "tests/protos"] reportDeprecated = false [tool.pytest.ini_options] @@ -105,7 +105,7 @@ include = "../../shared_tasks.toml" test = "pytest -n auto" mypy.default_item_type = "cmd" mypy.sequence = [ - "mypy --config-file ../../pyproject.toml --exclude src/autogen_core/application/protos src tests", + "mypy --config-file ../../pyproject.toml --exclude src/autogen_core/application/protos --exclude tests/protos src tests", "nbqa mypy docs/src --config-file ../../pyproject.toml", ] diff --git a/python/packages/autogen-core/tests/protos/serialization_test.proto b/python/packages/autogen-core/tests/protos/serialization_test.proto new file mode 100644 index 00000000000..611100ccde1 --- /dev/null +++ b/python/packages/autogen-core/tests/protos/serialization_test.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package agents; + +message ProtoMessage { + string message = 1; +} +message NestingProtoMessage { + string message = 1; + ProtoMessage nested = 2; +} \ No newline at end of file diff --git a/python/packages/autogen-core/tests/protos/serialization_test_pb2.py b/python/packages/autogen-core/tests/protos/serialization_test_pb2.py new file mode 100644 index 00000000000..3de431336b9 --- /dev/null +++ b/python/packages/autogen-core/tests/protos/serialization_test_pb2.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: serialization_test.proto +# Protobuf Python Version: 4.25.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x18serialization_test.proto\x12\x06\x61gents"\x1f\n\x0cProtoMessage\x12\x0f\n\x07message\x18\x01 \x01(\t"L\n\x13NestingProtoMessage\x12\x0f\n\x07message\x18\x01 \x01(\t\x12$\n\x06nested\x18\x02 \x01(\x0b\x32\x14.agents.ProtoMessageb\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "serialization_test_pb2", _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals["_PROTOMESSAGE"]._serialized_start = 36 + _globals["_PROTOMESSAGE"]._serialized_end = 67 + _globals["_NESTINGPROTOMESSAGE"]._serialized_start = 69 + _globals["_NESTINGPROTOMESSAGE"]._serialized_end = 145 +# @@protoc_insertion_point(module_scope) diff --git a/python/packages/autogen-core/tests/protos/serialization_test_pb2.pyi b/python/packages/autogen-core/tests/protos/serialization_test_pb2.pyi new file mode 100644 index 00000000000..b8a284663f6 --- /dev/null +++ b/python/packages/autogen-core/tests/protos/serialization_test_pb2.pyi @@ -0,0 +1,46 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import google.protobuf.descriptor +import google.protobuf.message +import typing + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing.final +class ProtoMessage(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + MESSAGE_FIELD_NUMBER: builtins.int + message: builtins.str + def __init__( + self, + *, + message: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["message", b"message"]) -> None: ... + +global___ProtoMessage = ProtoMessage + +@typing.final +class NestingProtoMessage(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + MESSAGE_FIELD_NUMBER: builtins.int + NESTED_FIELD_NUMBER: builtins.int + message: builtins.str + @property + def nested(self) -> global___ProtoMessage: ... + def __init__( + self, + *, + message: builtins.str = ..., + nested: global___ProtoMessage | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["nested", b"nested"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["message", b"message", "nested", b"nested"]) -> None: ... + +global___NestingProtoMessage = NestingProtoMessage diff --git a/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py b/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py new file mode 100644 index 00000000000..bf947056a2f --- /dev/null +++ b/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc diff --git a/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi b/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi new file mode 100644 index 00000000000..e8e2f7f6035 --- /dev/null +++ b/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi @@ -0,0 +1,16 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import abc +import collections.abc +import grpc +import grpc.aio +import typing + +_T = typing.TypeVar("_T") + +class _MaybeAsyncIterator(collections.abc.AsyncIterator[_T], collections.abc.Iterator[_T], metaclass=abc.ABCMeta): ... +class _ServicerContext(grpc.ServicerContext, grpc.aio.ServicerContext): # type: ignore[misc, type-arg] + ... diff --git a/python/packages/autogen-core/tests/test_serialization.py b/python/packages/autogen-core/tests/test_serialization.py index 3f3b0174c8b..6b5568411f6 100644 --- a/python/packages/autogen-core/tests/test_serialization.py +++ b/python/packages/autogen-core/tests/test_serialization.py @@ -11,6 +11,7 @@ from autogen_core.base._serialization import DataclassJsonMessageSerializer, PydanticJsonMessageSerializer from autogen_core.components import Image from PIL import Image as PILImage +from protos.serialization_test_pb2 import NestingProtoMessage, ProtoMessage from pydantic import BaseModel @@ -83,6 +84,36 @@ def test_nesting_dataclass_dataclass() -> None: serde.add_serializer(try_get_known_serializers_for_type(NestingDataclassMessage)) +def test_proto() -> None: + serde = SerializationRegistry() + serde.add_serializer(try_get_known_serializers_for_type(ProtoMessage)) + + message = ProtoMessage(message="hello") + name = serde.type_name(message) + # TODO: should be PROTO_DATA_CONTENT_TYPE + data = serde.serialize(message, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE) + assert name == "ProtoMessage" + # TODO: assert data == stuff + deserialized = serde.deserialize(data, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE) + assert deserialized == message + + +def test_nested_proto() -> None: + serde = SerializationRegistry() + serde.add_serializer(try_get_known_serializers_for_type(NestingProtoMessage)) + + message = NestingProtoMessage(message="hello", nested=ProtoMessage(message="world")) + name = serde.type_name(message) + + # TODO: should be PROTO_DATA_CONTENT_TYPE + data = serde.serialize(message, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE) + + # TODO: assert data == stuff + + deserialized = serde.deserialize(data, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE) + assert deserialized == message + + @dataclass class DataclassNestedUnionSyntaxOldMessage: message: Union[str, int] From a140a69979e3a1875a3eca85ed718f344b26a428 Mon Sep 17 00:00:00 2001 From: Peter Chang Date: Tue, 26 Nov 2024 13:53:24 -0500 Subject: [PATCH 2/2] proto file regeneration --- .github/workflows/checks.yml | 1 + .../tests/protos/serialization_test_pb2.py | 19 +++++++++---------- .../protos/serialization_test_pb2_grpc.py | 2 +- .../protos/serialization_test_pb2_grpc.pyi | 1 + python/pyproject.toml | 2 ++ 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 31869b86ccc..703c0749425 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -190,6 +190,7 @@ jobs: run: | source ${{ github.workspace }}/python/.venv/bin/activate poe gen-proto + poe gen-test-proto working-directory: ./python - name: Check if there are uncommited changes id: changes diff --git a/python/packages/autogen-core/tests/protos/serialization_test_pb2.py b/python/packages/autogen-core/tests/protos/serialization_test_pb2.py index 3de431336b9..ebc4bfee701 100644 --- a/python/packages/autogen-core/tests/protos/serialization_test_pb2.py +++ b/python/packages/autogen-core/tests/protos/serialization_test_pb2.py @@ -3,7 +3,6 @@ # source: serialization_test.proto # Protobuf Python Version: 4.25.1 """Generated protocol buffer code.""" - from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database @@ -13,17 +12,17 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x18serialization_test.proto\x12\x06\x61gents"\x1f\n\x0cProtoMessage\x12\x0f\n\x07message\x18\x01 \x01(\t"L\n\x13NestingProtoMessage\x12\x0f\n\x07message\x18\x01 \x01(\t\x12$\n\x06nested\x18\x02 \x01(\x0b\x32\x14.agents.ProtoMessageb\x06proto3' -) + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18serialization_test.proto\x12\x06\x61gents\"\x1f\n\x0cProtoMessage\x12\x0f\n\x07message\x18\x01 \x01(\t\"L\n\x13NestingProtoMessage\x12\x0f\n\x07message\x18\x01 \x01(\t\x12$\n\x06nested\x18\x02 \x01(\x0b\x32\x14.agents.ProtoMessageb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "serialization_test_pb2", _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'serialization_test_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _globals["_PROTOMESSAGE"]._serialized_start = 36 - _globals["_PROTOMESSAGE"]._serialized_end = 67 - _globals["_NESTINGPROTOMESSAGE"]._serialized_start = 69 - _globals["_NESTINGPROTOMESSAGE"]._serialized_end = 145 + DESCRIPTOR._options = None + _globals['_PROTOMESSAGE']._serialized_start=36 + _globals['_PROTOMESSAGE']._serialized_end=67 + _globals['_NESTINGPROTOMESSAGE']._serialized_start=69 + _globals['_NESTINGPROTOMESSAGE']._serialized_end=145 # @@protoc_insertion_point(module_scope) diff --git a/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py b/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py index bf947056a2f..2daafffebfc 100644 --- a/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py +++ b/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py @@ -1,4 +1,4 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc + diff --git a/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi b/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi index e8e2f7f6035..a6a9cff9dfd 100644 --- a/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi +++ b/python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi @@ -12,5 +12,6 @@ import typing _T = typing.TypeVar("_T") class _MaybeAsyncIterator(collections.abc.AsyncIterator[_T], collections.abc.Iterator[_T], metaclass=abc.ABCMeta): ... + class _ServicerContext(grpc.ServicerContext, grpc.aio.ServicerContext): # type: ignore[misc, type-arg] ... diff --git a/python/pyproject.toml b/python/pyproject.toml index 3b099db535e..e9b9753cfca 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -79,3 +79,5 @@ test = "python run_task_in_pkgs_if_exist.py test" check = ["fmt", "lint", "pyright", "mypy", "test"] gen-proto = "python -m grpc_tools.protoc --python_out=./packages/autogen-core/src/autogen_core/application/protos --grpc_python_out=./packages/autogen-core/src/autogen_core/application/protos --mypy_out=./packages/autogen-core/src/autogen_core/application/protos --mypy_grpc_out=./packages/autogen-core/src/autogen_core/application/protos --proto_path ../protos/ agent_worker.proto --proto_path ../protos/ cloudevent.proto" + +gen-test-proto = "python -m grpc_tools.protoc --python_out=./packages/autogen-core/tests/protos --grpc_python_out=./packages/autogen-core/tests/protos --mypy_out=./packages/autogen-core/tests/protos --mypy_grpc_out=./packages/autogen-core/tests/protos --proto_path ./packages/autogen-core/tests/protos serialization_test.proto" \ No newline at end of file