From d8acb9e6e15857011dadb988ca01bdcbbda6e532 Mon Sep 17 00:00:00 2001 From: Parth Mandaliya Date: Fri, 29 Sep 2023 17:31:52 +0530 Subject: [PATCH] Added weighted_average aggregation function under openfl.experimental.interface.{keras,torch}.aggregation_funtions Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya --- .../experimental/interface/keras/__init__.py | 7 + .../keras/aggregation_functions/__init__.py | 7 + .../aggregation_functions/weighted_average.py | 13 + .../experimental/interface/torch/__init__.py | 7 + .../torch/aggregation_functions/__init__.py | 7 + .../aggregation_functions/weighted_average.py | 77 +++++ openfl/protocols/aggregator_pb2.pyi | 148 ---------- openfl/protocols/base_pb2.pyi | 115 -------- openfl/protocols/director_pb2.pyi | 274 ------------------ setup.py | 5 +- 10 files changed, 122 insertions(+), 538 deletions(-) create mode 100644 openfl/experimental/interface/keras/__init__.py create mode 100644 openfl/experimental/interface/keras/aggregation_functions/__init__.py create mode 100644 openfl/experimental/interface/keras/aggregation_functions/weighted_average.py create mode 100644 openfl/experimental/interface/torch/__init__.py create mode 100644 openfl/experimental/interface/torch/aggregation_functions/__init__.py create mode 100644 openfl/experimental/interface/torch/aggregation_functions/weighted_average.py delete mode 100644 openfl/protocols/aggregator_pb2.pyi delete mode 100644 openfl/protocols/base_pb2.pyi delete mode 100644 openfl/protocols/director_pb2.pyi diff --git a/openfl/experimental/interface/keras/__init__.py b/openfl/experimental/interface/keras/__init__.py new file mode 100644 index 00000000000..1d7d84eb7f1 --- /dev/null +++ b/openfl/experimental/interface/keras/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.keras package.""" + +from .aggregation_functions import WeightedAverage + +__all__ = ["WeightedAverage", ] diff --git a/openfl/experimental/interface/keras/aggregation_functions/__init__.py b/openfl/experimental/interface/keras/aggregation_functions/__init__.py new file mode 100644 index 00000000000..94708487bcd --- /dev/null +++ b/openfl/experimental/interface/keras/aggregation_functions/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimenal.interface.keras.aggregation_functions package.""" + +from .weighted_average import WeightedAverage + +__all__ = ["WeightedAverage", ] diff --git a/openfl/experimental/interface/keras/aggregation_functions/weighted_average.py b/openfl/experimental/interface/keras/aggregation_functions/weighted_average.py new file mode 100644 index 00000000000..326e57aeced --- /dev/null +++ b/openfl/experimental/interface/keras/aggregation_functions/weighted_average.py @@ -0,0 +1,13 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.keras.aggregation_functions.weighted_average package.""" + + +class WeightedAverage: + """Weighted average aggregation for keras or tensorflow.""" + + def __init__(self) -> None: + """ + WeightedAverage class for Keras or Tensorflow library. + """ + raise NotImplementedError("WeightedAverage for keras will be implemented in the future.") diff --git a/openfl/experimental/interface/torch/__init__.py b/openfl/experimental/interface/torch/__init__.py new file mode 100644 index 00000000000..969f47b43a4 --- /dev/null +++ b/openfl/experimental/interface/torch/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.torch package.""" + +from .aggregation_functions import WeightedAverage + +__all__ = ["WeightedAverage", ] diff --git a/openfl/experimental/interface/torch/aggregation_functions/__init__.py b/openfl/experimental/interface/torch/aggregation_functions/__init__.py new file mode 100644 index 00000000000..2afa83b219f --- /dev/null +++ b/openfl/experimental/interface/torch/aggregation_functions/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimenal.interface.torch.aggregation_functions package.""" + +from .weighted_average import WeightedAverage + +__all__ = ["WeightedAverage", ] diff --git a/openfl/experimental/interface/torch/aggregation_functions/weighted_average.py b/openfl/experimental/interface/torch/aggregation_functions/weighted_average.py new file mode 100644 index 00000000000..a91cadfa0dd --- /dev/null +++ b/openfl/experimental/interface/torch/aggregation_functions/weighted_average.py @@ -0,0 +1,77 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.torch.aggregation_functions.weighted_average package.""" + +import collections +import numpy as np +import torch as pt + + +def weighted_average(tensors, weights): + """Compute weighted average.""" + return np.average(tensors, weights=weights, axis=0) + + +class WeightedAverage: + """Weighted average aggregation.""" + + def __call__(self, objects_list, weights_list) -> np.ndarray: + """ + Compute weighted average of models, optimizers, loss, or accuracy metrics. + For taking weighted average of optimizer do the following steps: + 1. Call "_get_optimizer_state" (openfl.federated.task.runner_pt._get_optimizer_state) + pass optimizer to it, to take optimizer state dictionary. + 2. Pass optimizer state dictionaries list to here. + 3. To set the weighted average optimizer state dictionary back to optimizer, + call "_set_optimizer_state" (openfl.federated.task.runner_pt._set_optimizer_state) + and pass optimizer, device, and optimizer dictionary received in step 2. + + Args: + objects_list: List of objects for which weighted average is to be computed. + - List of Model state dictionaries , or + - List of Metrics (Loss or accuracy), or + - List of optimizer state dictionaries (following steps need to be performed) + 1. Obtain optimizer state dictionary by invoking "_get_optimizer_state" + (openfl.federated.task.runner_pt._get_optimizer_state). + 2. Create a list of optimizer state dictionary obtained in step - 1 + Invoke WeightedAverage on this list. + 3. Invoke "_set_optimizer_state" to set weighted average of optimizer + state back to optimizer (openfl.federated.task.runner_pt._set_optimizer_state). + weights_list: Weight for each element in the list. + + Returns: + dict: For model or optimizer + float: For Loss or Accuracy metrics + """ + # Check the type of first element of tensors list + if type(objects_list[0]) in (dict, collections.OrderedDict): + optimizer = False + # If __opt_state_needed found then optimizer state dictionary is passed + if "__opt_state_needed" in objects_list[0]: + optimizer = True + # Remove __opt_state_needed from all state dictionary in list, and + # check if weightedaverage of optimizer can be taken. + for tensor in objects_list: + error_msg = "Optimizer is stateless, WeightedAverage cannot be taken" + assert tensor.pop("__opt_state_needed") == "true", error_msg + + tmp_list = [] + # # Take keys in order to rebuild the state dictionary taking keys back up + for tensor in objects_list: + # Append values of each state dictionary in list + # If type(value) is Tensor then it needs to be detached + tmp_list.append(np.array([value.detach() if isinstance(value, pt.Tensor) else value + for value in tensor.values()], dtype=object)) + # Take weighted average of list of arrays + # new_params passed is weighted average of each array in tmp_list + new_params = weighted_average(tmp_list, weights_list) + new_state = {} + # Take weighted average parameters and building a dictionary + for i, k in enumerate(objects_list[0].keys()): + if optimizer: + new_state[k] = new_params[i] + else: + new_state[k] = pt.from_numpy(new_params[i].numpy()) + return new_state + else: + return weighted_average(objects_list, weights_list) diff --git a/openfl/protocols/aggregator_pb2.pyi b/openfl/protocols/aggregator_pb2.pyi deleted file mode 100644 index 7857a22896d..00000000000 --- a/openfl/protocols/aggregator_pb2.pyi +++ /dev/null @@ -1,148 +0,0 @@ -from openfl.protocols import base_pb2 as _base_pb2 -from google.protobuf.internal import containers as _containers -from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union - -DESCRIPTOR: _descriptor.FileDescriptor - -class GetAggregatedTensorRequest(_message.Message): - __slots__ = ["header", "report", "require_lossless", "round_number", "tags", "tensor_name"] - HEADER_FIELD_NUMBER: _ClassVar[int] - REPORT_FIELD_NUMBER: _ClassVar[int] - REQUIRE_LOSSLESS_FIELD_NUMBER: _ClassVar[int] - ROUND_NUMBER_FIELD_NUMBER: _ClassVar[int] - TAGS_FIELD_NUMBER: _ClassVar[int] - TENSOR_NAME_FIELD_NUMBER: _ClassVar[int] - header: MessageHeader - report: bool - require_lossless: bool - round_number: int - tags: _containers.RepeatedScalarFieldContainer[str] - tensor_name: str - def __init__(self, header: _Optional[_Union[MessageHeader, _Mapping]] = ..., tensor_name: _Optional[str] = ..., round_number: _Optional[int] = ..., report: bool = ..., tags: _Optional[_Iterable[str]] = ..., require_lossless: bool = ...) -> None: ... - -class GetAggregatedTensorResponse(_message.Message): - __slots__ = ["header", "round_number", "tensor"] - HEADER_FIELD_NUMBER: _ClassVar[int] - ROUND_NUMBER_FIELD_NUMBER: _ClassVar[int] - TENSOR_FIELD_NUMBER: _ClassVar[int] - header: MessageHeader - round_number: int - tensor: _base_pb2.NamedTensor - def __init__(self, header: _Optional[_Union[MessageHeader, _Mapping]] = ..., round_number: _Optional[int] = ..., tensor: _Optional[_Union[_base_pb2.NamedTensor, _Mapping]] = ...) -> None: ... - -class GetExperimentDescriptionRequest(_message.Message): - __slots__ = ["name"] - NAME_FIELD_NUMBER: _ClassVar[int] - name: str - def __init__(self, name: _Optional[str] = ...) -> None: ... - -class GetExperimentDescriptionResponse(_message.Message): - __slots__ = ["experiment"] - EXPERIMENT_FIELD_NUMBER: _ClassVar[int] - experiment: _base_pb2.ExperimentDescription - def __init__(self, experiment: _Optional[_Union[_base_pb2.ExperimentDescription, _Mapping]] = ...) -> None: ... - -class GetMetricStreamRequest(_message.Message): - __slots__ = ["experiment_name"] - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - experiment_name: str - def __init__(self, experiment_name: _Optional[str] = ...) -> None: ... - -class GetMetricStreamResponse(_message.Message): - __slots__ = ["metric_name", "metric_origin", "metric_value", "round", "task_name"] - METRIC_NAME_FIELD_NUMBER: _ClassVar[int] - METRIC_ORIGIN_FIELD_NUMBER: _ClassVar[int] - METRIC_VALUE_FIELD_NUMBER: _ClassVar[int] - ROUND_FIELD_NUMBER: _ClassVar[int] - TASK_NAME_FIELD_NUMBER: _ClassVar[int] - metric_name: str - metric_origin: str - metric_value: float - round: int - task_name: str - def __init__(self, metric_origin: _Optional[str] = ..., task_name: _Optional[str] = ..., metric_name: _Optional[str] = ..., metric_value: _Optional[float] = ..., round: _Optional[int] = ...) -> None: ... - -class GetTasksRequest(_message.Message): - __slots__ = ["header"] - HEADER_FIELD_NUMBER: _ClassVar[int] - header: MessageHeader - def __init__(self, header: _Optional[_Union[MessageHeader, _Mapping]] = ...) -> None: ... - -class GetTasksResponse(_message.Message): - __slots__ = ["header", "quit", "round_number", "sleep_time", "tasks"] - HEADER_FIELD_NUMBER: _ClassVar[int] - QUIT_FIELD_NUMBER: _ClassVar[int] - ROUND_NUMBER_FIELD_NUMBER: _ClassVar[int] - SLEEP_TIME_FIELD_NUMBER: _ClassVar[int] - TASKS_FIELD_NUMBER: _ClassVar[int] - header: MessageHeader - quit: bool - round_number: int - sleep_time: int - tasks: _containers.RepeatedCompositeFieldContainer[Task] - def __init__(self, header: _Optional[_Union[MessageHeader, _Mapping]] = ..., round_number: _Optional[int] = ..., tasks: _Optional[_Iterable[_Union[Task, _Mapping]]] = ..., sleep_time: _Optional[int] = ..., quit: bool = ...) -> None: ... - -class GetTrainedModelRequest(_message.Message): - __slots__ = ["experiment_name", "model_type"] - class ModelType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] - BEST_MODEL: GetTrainedModelRequest.ModelType - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - LAST_MODEL: GetTrainedModelRequest.ModelType - MODEL_TYPE_FIELD_NUMBER: _ClassVar[int] - experiment_name: str - model_type: GetTrainedModelRequest.ModelType - def __init__(self, experiment_name: _Optional[str] = ..., model_type: _Optional[_Union[GetTrainedModelRequest.ModelType, str]] = ...) -> None: ... - -class MessageHeader(_message.Message): - __slots__ = ["federation_uuid", "receiver", "sender", "single_col_cert_common_name"] - FEDERATION_UUID_FIELD_NUMBER: _ClassVar[int] - RECEIVER_FIELD_NUMBER: _ClassVar[int] - SENDER_FIELD_NUMBER: _ClassVar[int] - SINGLE_COL_CERT_COMMON_NAME_FIELD_NUMBER: _ClassVar[int] - federation_uuid: str - receiver: str - sender: str - single_col_cert_common_name: str - def __init__(self, sender: _Optional[str] = ..., receiver: _Optional[str] = ..., federation_uuid: _Optional[str] = ..., single_col_cert_common_name: _Optional[str] = ...) -> None: ... - -class SendLocalTaskResultsResponse(_message.Message): - __slots__ = ["header"] - HEADER_FIELD_NUMBER: _ClassVar[int] - header: MessageHeader - def __init__(self, header: _Optional[_Union[MessageHeader, _Mapping]] = ...) -> None: ... - -class Task(_message.Message): - __slots__ = ["apply_local", "function_name", "name", "task_type"] - APPLY_LOCAL_FIELD_NUMBER: _ClassVar[int] - FUNCTION_NAME_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - TASK_TYPE_FIELD_NUMBER: _ClassVar[int] - apply_local: bool - function_name: str - name: str - task_type: str - def __init__(self, name: _Optional[str] = ..., function_name: _Optional[str] = ..., task_type: _Optional[str] = ..., apply_local: bool = ...) -> None: ... - -class TaskResults(_message.Message): - __slots__ = ["data_size", "header", "round_number", "task_name", "tensors"] - DATA_SIZE_FIELD_NUMBER: _ClassVar[int] - HEADER_FIELD_NUMBER: _ClassVar[int] - ROUND_NUMBER_FIELD_NUMBER: _ClassVar[int] - TASK_NAME_FIELD_NUMBER: _ClassVar[int] - TENSORS_FIELD_NUMBER: _ClassVar[int] - data_size: int - header: MessageHeader - round_number: int - task_name: str - tensors: _containers.RepeatedCompositeFieldContainer[_base_pb2.NamedTensor] - def __init__(self, header: _Optional[_Union[MessageHeader, _Mapping]] = ..., round_number: _Optional[int] = ..., task_name: _Optional[str] = ..., data_size: _Optional[int] = ..., tensors: _Optional[_Iterable[_Union[_base_pb2.NamedTensor, _Mapping]]] = ...) -> None: ... - -class TrainedModelResponse(_message.Message): - __slots__ = ["model_proto"] - MODEL_PROTO_FIELD_NUMBER: _ClassVar[int] - model_proto: _base_pb2.ModelProto - def __init__(self, model_proto: _Optional[_Union[_base_pb2.ModelProto, _Mapping]] = ...) -> None: ... diff --git a/openfl/protocols/base_pb2.pyi b/openfl/protocols/base_pb2.pyi deleted file mode 100644 index 83dffbc51cd..00000000000 --- a/openfl/protocols/base_pb2.pyi +++ /dev/null @@ -1,115 +0,0 @@ -from google.protobuf.internal import containers as _containers -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union - -DESCRIPTOR: _descriptor.FileDescriptor - -class CollaboratorDescription(_message.Message): - __slots__ = ["current_task", "name", "next_task", "progress", "round", "status"] - CURRENT_TASK_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - NEXT_TASK_FIELD_NUMBER: _ClassVar[int] - PROGRESS_FIELD_NUMBER: _ClassVar[int] - ROUND_FIELD_NUMBER: _ClassVar[int] - STATUS_FIELD_NUMBER: _ClassVar[int] - current_task: str - name: str - next_task: str - progress: float - round: int - status: str - def __init__(self, name: _Optional[str] = ..., status: _Optional[str] = ..., progress: _Optional[float] = ..., round: _Optional[int] = ..., current_task: _Optional[str] = ..., next_task: _Optional[str] = ...) -> None: ... - -class DataStream(_message.Message): - __slots__ = ["npbytes", "size"] - NPBYTES_FIELD_NUMBER: _ClassVar[int] - SIZE_FIELD_NUMBER: _ClassVar[int] - npbytes: bytes - size: int - def __init__(self, size: _Optional[int] = ..., npbytes: _Optional[bytes] = ...) -> None: ... - -class DownloadStatus(_message.Message): - __slots__ = ["name", "status"] - NAME_FIELD_NUMBER: _ClassVar[int] - STATUS_FIELD_NUMBER: _ClassVar[int] - name: str - status: str - def __init__(self, name: _Optional[str] = ..., status: _Optional[str] = ...) -> None: ... - -class DownloadStatuses(_message.Message): - __slots__ = ["logs", "models"] - LOGS_FIELD_NUMBER: _ClassVar[int] - MODELS_FIELD_NUMBER: _ClassVar[int] - logs: _containers.RepeatedCompositeFieldContainer[DownloadStatus] - models: _containers.RepeatedCompositeFieldContainer[DownloadStatus] - def __init__(self, models: _Optional[_Iterable[_Union[DownloadStatus, _Mapping]]] = ..., logs: _Optional[_Iterable[_Union[DownloadStatus, _Mapping]]] = ...) -> None: ... - -class ExperimentDescription(_message.Message): - __slots__ = ["collaborators", "current_round", "download_statuses", "name", "progress", "status", "tasks", "total_rounds"] - COLLABORATORS_FIELD_NUMBER: _ClassVar[int] - CURRENT_ROUND_FIELD_NUMBER: _ClassVar[int] - DOWNLOAD_STATUSES_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - PROGRESS_FIELD_NUMBER: _ClassVar[int] - STATUS_FIELD_NUMBER: _ClassVar[int] - TASKS_FIELD_NUMBER: _ClassVar[int] - TOTAL_ROUNDS_FIELD_NUMBER: _ClassVar[int] - collaborators: _containers.RepeatedCompositeFieldContainer[CollaboratorDescription] - current_round: int - download_statuses: DownloadStatuses - name: str - progress: float - status: str - tasks: _containers.RepeatedCompositeFieldContainer[TaskDescription] - total_rounds: int - def __init__(self, name: _Optional[str] = ..., status: _Optional[str] = ..., progress: _Optional[float] = ..., total_rounds: _Optional[int] = ..., current_round: _Optional[int] = ..., download_statuses: _Optional[_Union[DownloadStatuses, _Mapping]] = ..., collaborators: _Optional[_Iterable[_Union[CollaboratorDescription, _Mapping]]] = ..., tasks: _Optional[_Iterable[_Union[TaskDescription, _Mapping]]] = ...) -> None: ... - -class MetadataProto(_message.Message): - __slots__ = ["bool_list", "int_list", "int_to_float"] - class IntToFloatEntry(_message.Message): - __slots__ = ["key", "value"] - KEY_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - key: int - value: float - def __init__(self, key: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... - BOOL_LIST_FIELD_NUMBER: _ClassVar[int] - INT_LIST_FIELD_NUMBER: _ClassVar[int] - INT_TO_FLOAT_FIELD_NUMBER: _ClassVar[int] - bool_list: _containers.RepeatedScalarFieldContainer[bool] - int_list: _containers.RepeatedScalarFieldContainer[int] - int_to_float: _containers.ScalarMap[int, float] - def __init__(self, int_to_float: _Optional[_Mapping[int, float]] = ..., int_list: _Optional[_Iterable[int]] = ..., bool_list: _Optional[_Iterable[bool]] = ...) -> None: ... - -class ModelProto(_message.Message): - __slots__ = ["tensors"] - TENSORS_FIELD_NUMBER: _ClassVar[int] - tensors: _containers.RepeatedCompositeFieldContainer[NamedTensor] - def __init__(self, tensors: _Optional[_Iterable[_Union[NamedTensor, _Mapping]]] = ...) -> None: ... - -class NamedTensor(_message.Message): - __slots__ = ["data_bytes", "lossless", "name", "report", "round_number", "tags", "transformer_metadata"] - DATA_BYTES_FIELD_NUMBER: _ClassVar[int] - LOSSLESS_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - REPORT_FIELD_NUMBER: _ClassVar[int] - ROUND_NUMBER_FIELD_NUMBER: _ClassVar[int] - TAGS_FIELD_NUMBER: _ClassVar[int] - TRANSFORMER_METADATA_FIELD_NUMBER: _ClassVar[int] - data_bytes: bytes - lossless: bool - name: str - report: bool - round_number: int - tags: _containers.RepeatedScalarFieldContainer[str] - transformer_metadata: _containers.RepeatedCompositeFieldContainer[MetadataProto] - def __init__(self, name: _Optional[str] = ..., round_number: _Optional[int] = ..., lossless: bool = ..., report: bool = ..., tags: _Optional[_Iterable[str]] = ..., transformer_metadata: _Optional[_Iterable[_Union[MetadataProto, _Mapping]]] = ..., data_bytes: _Optional[bytes] = ...) -> None: ... - -class TaskDescription(_message.Message): - __slots__ = ["description", "name"] - DESCRIPTION_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - description: str - name: str - def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ...) -> None: ... diff --git a/openfl/protocols/director_pb2.pyi b/openfl/protocols/director_pb2.pyi deleted file mode 100644 index 825a667ec64..00000000000 --- a/openfl/protocols/director_pb2.pyi +++ /dev/null @@ -1,274 +0,0 @@ -from google.protobuf import timestamp_pb2 as _timestamp_pb2 -from google.protobuf import duration_pb2 as _duration_pb2 -from openfl.protocols import base_pb2 as _base_pb2 -from google.protobuf.internal import containers as _containers -from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union - -DESCRIPTOR: _descriptor.FileDescriptor - -class CudaDeviceInfo(_message.Message): - __slots__ = ["cuda_driver_version", "cuda_version", "device_utilization", "index", "memory_total", "memory_utilized", "name"] - CUDA_DRIVER_VERSION_FIELD_NUMBER: _ClassVar[int] - CUDA_VERSION_FIELD_NUMBER: _ClassVar[int] - DEVICE_UTILIZATION_FIELD_NUMBER: _ClassVar[int] - INDEX_FIELD_NUMBER: _ClassVar[int] - MEMORY_TOTAL_FIELD_NUMBER: _ClassVar[int] - MEMORY_UTILIZED_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - cuda_driver_version: str - cuda_version: str - device_utilization: str - index: int - memory_total: int - memory_utilized: int - name: str - def __init__(self, index: _Optional[int] = ..., memory_total: _Optional[int] = ..., memory_utilized: _Optional[int] = ..., device_utilization: _Optional[str] = ..., cuda_driver_version: _Optional[str] = ..., cuda_version: _Optional[str] = ..., name: _Optional[str] = ...) -> None: ... - -class EnvoyInfo(_message.Message): - __slots__ = ["experiment_name", "is_experiment_running", "is_online", "last_updated", "shard_info", "valid_duration"] - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - IS_EXPERIMENT_RUNNING_FIELD_NUMBER: _ClassVar[int] - IS_ONLINE_FIELD_NUMBER: _ClassVar[int] - LAST_UPDATED_FIELD_NUMBER: _ClassVar[int] - SHARD_INFO_FIELD_NUMBER: _ClassVar[int] - VALID_DURATION_FIELD_NUMBER: _ClassVar[int] - experiment_name: str - is_experiment_running: bool - is_online: bool - last_updated: _timestamp_pb2.Timestamp - shard_info: ShardInfo - valid_duration: _duration_pb2.Duration - def __init__(self, shard_info: _Optional[_Union[ShardInfo, _Mapping]] = ..., is_online: bool = ..., is_experiment_running: bool = ..., last_updated: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., valid_duration: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., experiment_name: _Optional[str] = ...) -> None: ... - -class ExperimentData(_message.Message): - __slots__ = ["npbytes", "size"] - NPBYTES_FIELD_NUMBER: _ClassVar[int] - SIZE_FIELD_NUMBER: _ClassVar[int] - npbytes: bytes - size: int - def __init__(self, size: _Optional[int] = ..., npbytes: _Optional[bytes] = ...) -> None: ... - -class ExperimentInfo(_message.Message): - __slots__ = ["collaborator_names", "experiment_data", "model_proto", "name"] - COLLABORATOR_NAMES_FIELD_NUMBER: _ClassVar[int] - EXPERIMENT_DATA_FIELD_NUMBER: _ClassVar[int] - MODEL_PROTO_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - collaborator_names: _containers.RepeatedScalarFieldContainer[str] - experiment_data: ExperimentData - model_proto: _base_pb2.ModelProto - name: str - def __init__(self, name: _Optional[str] = ..., collaborator_names: _Optional[_Iterable[str]] = ..., experiment_data: _Optional[_Union[ExperimentData, _Mapping]] = ..., model_proto: _Optional[_Union[_base_pb2.ModelProto, _Mapping]] = ...) -> None: ... - -class ExperimentListItem(_message.Message): - __slots__ = ["collaborators_amount", "name", "progress", "status", "tasks_amount"] - COLLABORATORS_AMOUNT_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - PROGRESS_FIELD_NUMBER: _ClassVar[int] - STATUS_FIELD_NUMBER: _ClassVar[int] - TASKS_AMOUNT_FIELD_NUMBER: _ClassVar[int] - collaborators_amount: int - name: str - progress: float - status: str - tasks_amount: int - def __init__(self, name: _Optional[str] = ..., status: _Optional[str] = ..., collaborators_amount: _Optional[int] = ..., tasks_amount: _Optional[int] = ..., progress: _Optional[float] = ...) -> None: ... - -class GetDatasetInfoRequest(_message.Message): - __slots__ = [] - def __init__(self) -> None: ... - -class GetDatasetInfoResponse(_message.Message): - __slots__ = ["shard_info"] - SHARD_INFO_FIELD_NUMBER: _ClassVar[int] - shard_info: ShardInfo - def __init__(self, shard_info: _Optional[_Union[ShardInfo, _Mapping]] = ...) -> None: ... - -class GetEnvoysRequest(_message.Message): - __slots__ = [] - def __init__(self) -> None: ... - -class GetEnvoysResponse(_message.Message): - __slots__ = ["envoy_infos"] - ENVOY_INFOS_FIELD_NUMBER: _ClassVar[int] - envoy_infos: _containers.RepeatedCompositeFieldContainer[EnvoyInfo] - def __init__(self, envoy_infos: _Optional[_Iterable[_Union[EnvoyInfo, _Mapping]]] = ...) -> None: ... - -class GetExperimentDataRequest(_message.Message): - __slots__ = ["collaborator_name", "experiment_name"] - COLLABORATOR_NAME_FIELD_NUMBER: _ClassVar[int] - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - collaborator_name: str - experiment_name: str - def __init__(self, experiment_name: _Optional[str] = ..., collaborator_name: _Optional[str] = ...) -> None: ... - -class GetExperimentDescriptionRequest(_message.Message): - __slots__ = ["name"] - NAME_FIELD_NUMBER: _ClassVar[int] - name: str - def __init__(self, name: _Optional[str] = ...) -> None: ... - -class GetExperimentDescriptionResponse(_message.Message): - __slots__ = ["experiment"] - EXPERIMENT_FIELD_NUMBER: _ClassVar[int] - experiment: _base_pb2.ExperimentDescription - def __init__(self, experiment: _Optional[_Union[_base_pb2.ExperimentDescription, _Mapping]] = ...) -> None: ... - -class GetExperimentStatusRequest(_message.Message): - __slots__ = ["experiment_name"] - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - experiment_name: str - def __init__(self, experiment_name: _Optional[str] = ...) -> None: ... - -class GetExperimentStatusResponse(_message.Message): - __slots__ = ["experiment_status"] - EXPERIMENT_STATUS_FIELD_NUMBER: _ClassVar[int] - experiment_status: str - def __init__(self, experiment_status: _Optional[str] = ...) -> None: ... - -class GetExperimentsListRequest(_message.Message): - __slots__ = [] - def __init__(self) -> None: ... - -class GetExperimentsListResponse(_message.Message): - __slots__ = ["experiments"] - EXPERIMENTS_FIELD_NUMBER: _ClassVar[int] - experiments: _containers.RepeatedCompositeFieldContainer[ExperimentListItem] - def __init__(self, experiments: _Optional[_Iterable[_Union[ExperimentListItem, _Mapping]]] = ...) -> None: ... - -class GetMetricStreamRequest(_message.Message): - __slots__ = ["experiment_name"] - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - experiment_name: str - def __init__(self, experiment_name: _Optional[str] = ...) -> None: ... - -class GetMetricStreamResponse(_message.Message): - __slots__ = ["metric_name", "metric_origin", "metric_value", "round", "task_name"] - METRIC_NAME_FIELD_NUMBER: _ClassVar[int] - METRIC_ORIGIN_FIELD_NUMBER: _ClassVar[int] - METRIC_VALUE_FIELD_NUMBER: _ClassVar[int] - ROUND_FIELD_NUMBER: _ClassVar[int] - TASK_NAME_FIELD_NUMBER: _ClassVar[int] - metric_name: str - metric_origin: str - metric_value: float - round: int - task_name: str - def __init__(self, metric_origin: _Optional[str] = ..., task_name: _Optional[str] = ..., metric_name: _Optional[str] = ..., metric_value: _Optional[float] = ..., round: _Optional[int] = ...) -> None: ... - -class GetTrainedModelRequest(_message.Message): - __slots__ = ["experiment_name", "model_type"] - class ModelType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] - BEST_MODEL: GetTrainedModelRequest.ModelType - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - LAST_MODEL: GetTrainedModelRequest.ModelType - MODEL_TYPE_FIELD_NUMBER: _ClassVar[int] - experiment_name: str - model_type: GetTrainedModelRequest.ModelType - def __init__(self, experiment_name: _Optional[str] = ..., model_type: _Optional[_Union[GetTrainedModelRequest.ModelType, str]] = ...) -> None: ... - -class NodeInfo(_message.Message): - __slots__ = ["cuda_devices", "name"] - CUDA_DEVICES_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - cuda_devices: _containers.RepeatedCompositeFieldContainer[CudaDeviceInfo] - name: str - def __init__(self, name: _Optional[str] = ..., cuda_devices: _Optional[_Iterable[_Union[CudaDeviceInfo, _Mapping]]] = ...) -> None: ... - -class RemoveExperimentRequest(_message.Message): - __slots__ = ["experiment_name"] - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - experiment_name: str - def __init__(self, experiment_name: _Optional[str] = ...) -> None: ... - -class RemoveExperimentResponse(_message.Message): - __slots__ = ["acknowledgement"] - ACKNOWLEDGEMENT_FIELD_NUMBER: _ClassVar[int] - acknowledgement: bool - def __init__(self, acknowledgement: bool = ...) -> None: ... - -class SetExperimentFailedRequest(_message.Message): - __slots__ = ["collaborator_name", "error_code", "error_description", "experiment_name"] - COLLABORATOR_NAME_FIELD_NUMBER: _ClassVar[int] - ERROR_CODE_FIELD_NUMBER: _ClassVar[int] - ERROR_DESCRIPTION_FIELD_NUMBER: _ClassVar[int] - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - collaborator_name: str - error_code: int - error_description: str - experiment_name: str - def __init__(self, experiment_name: _Optional[str] = ..., collaborator_name: _Optional[str] = ..., error_code: _Optional[int] = ..., error_description: _Optional[str] = ...) -> None: ... - -class SetExperimentFailedResponse(_message.Message): - __slots__ = [] - def __init__(self) -> None: ... - -class SetNewExperimentResponse(_message.Message): - __slots__ = ["accepted"] - ACCEPTED_FIELD_NUMBER: _ClassVar[int] - accepted: bool - def __init__(self, accepted: bool = ...) -> None: ... - -class ShardInfo(_message.Message): - __slots__ = ["n_samples", "node_info", "sample_shape", "shard_description", "target_shape"] - NODE_INFO_FIELD_NUMBER: _ClassVar[int] - N_SAMPLES_FIELD_NUMBER: _ClassVar[int] - SAMPLE_SHAPE_FIELD_NUMBER: _ClassVar[int] - SHARD_DESCRIPTION_FIELD_NUMBER: _ClassVar[int] - TARGET_SHAPE_FIELD_NUMBER: _ClassVar[int] - n_samples: int - node_info: NodeInfo - sample_shape: _containers.RepeatedScalarFieldContainer[str] - shard_description: str - target_shape: _containers.RepeatedScalarFieldContainer[str] - def __init__(self, node_info: _Optional[_Union[NodeInfo, _Mapping]] = ..., shard_description: _Optional[str] = ..., n_samples: _Optional[int] = ..., sample_shape: _Optional[_Iterable[str]] = ..., target_shape: _Optional[_Iterable[str]] = ...) -> None: ... - -class TrainedModelResponse(_message.Message): - __slots__ = ["model_proto"] - MODEL_PROTO_FIELD_NUMBER: _ClassVar[int] - model_proto: _base_pb2.ModelProto - def __init__(self, model_proto: _Optional[_Union[_base_pb2.ModelProto, _Mapping]] = ...) -> None: ... - -class UpdateEnvoyStatusRequest(_message.Message): - __slots__ = ["cuda_devices", "is_experiment_running", "name"] - CUDA_DEVICES_FIELD_NUMBER: _ClassVar[int] - IS_EXPERIMENT_RUNNING_FIELD_NUMBER: _ClassVar[int] - NAME_FIELD_NUMBER: _ClassVar[int] - cuda_devices: _containers.RepeatedCompositeFieldContainer[CudaDeviceInfo] - is_experiment_running: bool - name: str - def __init__(self, name: _Optional[str] = ..., is_experiment_running: bool = ..., cuda_devices: _Optional[_Iterable[_Union[CudaDeviceInfo, _Mapping]]] = ...) -> None: ... - -class UpdateEnvoyStatusResponse(_message.Message): - __slots__ = ["health_check_period"] - HEALTH_CHECK_PERIOD_FIELD_NUMBER: _ClassVar[int] - health_check_period: _duration_pb2.Duration - def __init__(self, health_check_period: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ...) -> None: ... - -class UpdateShardInfoRequest(_message.Message): - __slots__ = ["shard_info"] - SHARD_INFO_FIELD_NUMBER: _ClassVar[int] - shard_info: ShardInfo - def __init__(self, shard_info: _Optional[_Union[ShardInfo, _Mapping]] = ...) -> None: ... - -class UpdateShardInfoResponse(_message.Message): - __slots__ = ["accepted"] - ACCEPTED_FIELD_NUMBER: _ClassVar[int] - accepted: bool - def __init__(self, accepted: bool = ...) -> None: ... - -class WaitExperimentRequest(_message.Message): - __slots__ = ["collaborator_name"] - COLLABORATOR_NAME_FIELD_NUMBER: _ClassVar[int] - collaborator_name: str - def __init__(self, collaborator_name: _Optional[str] = ...) -> None: ... - -class WaitExperimentResponse(_message.Message): - __slots__ = ["experiment_name"] - EXPERIMENT_NAME_FIELD_NUMBER: _ClassVar[int] - experiment_name: str - def __init__(self, experiment_name: _Optional[str] = ...) -> None: ... diff --git a/setup.py b/setup.py index 087cef4161a..08cc0433d6b 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,6 @@ def run(self): 'openfl.databases', 'openfl.databases.utilities', 'openfl.experimental', - # 'openfl.experimental.workspace_creator', 'openfl.experimental.workspace_builder', 'openfl.experimental.federated', 'openfl.experimental.federated.plan', @@ -111,6 +110,10 @@ def run(self): 'openfl.experimental.component.collaborator', 'openfl.experimental.interface.cli', 'openfl.experimental.interface', + 'openfl.experimental.interface.keras', + 'openfl.experimental.interface.keras.aggregation_functions', + 'openfl.experimental.interface.torch', + 'openfl.experimental.interface.torch.aggregation_functions', 'openfl.experimental.placement', 'openfl.experimental.runtime', 'openfl.experimental.protocols',