From 6db01a04ca5df23e9fcfbfd3b7c1f4763b782f83 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Thu, 18 Jan 2024 21:02:30 +0000 Subject: [PATCH 01/18] v0 --- src/py/flwr/common/flowercontext.py | 79 +++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/py/flwr/common/flowercontext.py diff --git a/src/py/flwr/common/flowercontext.py b/src/py/flwr/common/flowercontext.py new file mode 100644 index 000000000000..21f85fa16b20 --- /dev/null +++ b/src/py/flwr/common/flowercontext.py @@ -0,0 +1,79 @@ +# Copyright 2024 Flower Labs GmbH. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""FlowerContext and Metadata.""" + + +from dataclasses import dataclass + +from .recordset import RecordSet + + +@dataclass +class Metadata: + """A dataclass holding metadata associated. + + with the current task. + + Parameters + ---------- + task_id : str + An identifier for the current task. + run_id : int + An identifier for the current run. + task_type : str + A string that encodes a action to be executed on + the receiving end. + group_id : str + An identifier for grouping runs. In some settings + this is used as the FL round. + ttl : str + Time-to-live for this run + """ + + task_id: str + run_id: int + task_type: str + group_id: str + ttl: str + + +@dataclass +class FlowerContext: + """A dataclass representing the state of your application. + + from the viewpoint of the entity (e.g. a client, the driver) + making use of a given FlowerContext object. + + Parameters + ---------- + in_message : RecordSet + Holds records sent by another entity (e.g. sent by the server-side + logic to a client, or vice-versa) + out_message : RecordSet + Holds records added by the current entity. This `RecordSet` will + be sent out (e.g. back to the server-side for aggregation of + parameter, metrics, etc) + local : RecordSet + Holds record added by the current entity and that will stay local. + This can be used as an intermediate storage or scratchpad when + executing middleware layers. + metadata : Metadata + A dataclass including information about the task to be executed. + """ + + in_message: RecordSet + out_message: RecordSet + local: RecordSet + metadata: Metadata From a816f38692bad3f6e473f8e361f50b3dba4247c6 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Thu, 18 Jan 2024 21:13:57 +0000 Subject: [PATCH 02/18] updates to docstrings --- src/py/flwr/common/flowercontext.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/py/flwr/common/flowercontext.py b/src/py/flwr/common/flowercontext.py index 21f85fa16b20..f5dcc80d742f 100644 --- a/src/py/flwr/common/flowercontext.py +++ b/src/py/flwr/common/flowercontext.py @@ -22,9 +22,7 @@ @dataclass class Metadata: - """A dataclass holding metadata associated. - - with the current task. + """A dataclass holding metadata associated with the current task. Parameters ---------- @@ -39,7 +37,7 @@ class Metadata: An identifier for grouping runs. In some settings this is used as the FL round. ttl : str - Time-to-live for this run + Time-to-live for this run. """ task_id: str @@ -51,10 +49,7 @@ class Metadata: @dataclass class FlowerContext: - """A dataclass representing the state of your application. - - from the viewpoint of the entity (e.g. a client, the driver) - making use of a given FlowerContext object. + """State of your application from the viewpoint of the entity using it. Parameters ---------- @@ -64,11 +59,14 @@ class FlowerContext: out_message : RecordSet Holds records added by the current entity. This `RecordSet` will be sent out (e.g. back to the server-side for aggregation of - parameter, metrics, etc) + parameter, or to the client to perform a certain task) local : RecordSet Holds record added by the current entity and that will stay local. + This means that the data it holds will never leave the system it's running fom. This can be used as an intermediate storage or scratchpad when - executing middleware layers. + executing middleware layers. It can also be used as a memory to access + at different points during the lifecycle of this entity (e.g. across + multiple rounds) metadata : Metadata A dataclass including information about the task to be executed. """ From 775f09cb7151c9ab4ceef3378cc85c64a9dde9b8 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Fri, 19 Jan 2024 08:57:36 +0000 Subject: [PATCH 03/18] wip --- src/py/flwr/common/recordset_utils.py | 57 ++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_utils.py index c1e724fa2758..3f6dfca09ffb 100644 --- a/src/py/flwr/common/recordset_utils.py +++ b/src/py/flwr/common/recordset_utils.py @@ -15,10 +15,23 @@ """RecordSet utilities.""" -from typing import OrderedDict +from typing import OrderedDict, Dict +from .configsrecord import ConfigsRecord from .parametersrecord import Array, ParametersRecord -from .typing import Parameters +from .recordset import RecordSet +from .typing import ( + EvaluateIns, + EvaluateRes, + FitIns, + FitRes, + GetParametersIns, + GetParametersRes, + GetPropertiesIns, + GetPropertiesRes, + Parameters, + ConfigsRecordValues +) def parametersrecord_to_parameters( @@ -85,3 +98,43 @@ def parameters_to_parametersrecord( ) return p_record + + +def fuse_configsrecord_data(configs: ConfigsRecord) -> Dict[str, ConfigsRecordValues]: + """Fuse all config entries into a single dictionary.""" + + + +def recordset_to_fit_ins(recordset: RecordSet) -> FitIns: + + config = recordset.configs + + return FitIns() + + +def fit_res_to_recordset(fitres: FitRes) -> RecordSet: + return RecordSet() + + +def recodset_to_evaluate_ins(recordset: RecordSet) -> EvaluateIns: + return EvaluateIns() + + +def evaluate_res_to_recordset(evaluateres: EvaluateRes) -> RecordSet: + return RecordSet() + + +def recordset_to_getparameters_ins(recordset: RecordSet) -> GetParametersIns: + return GetParametersIns + + +def getparameters_res_to_recordset(getparametersres: GetParametersRes) -> RecordSet: + return RecordSet() + + +def recordset_to_getproperties_ins(recordset: RecordSet) -> GetPropertiesIns: + return GetPropertiesIns() + + +def getproperties_res_to_recorset(getpropertiesres: GetPropertiesRes) -> RecordSet: + return RecordSet() From 5908a64fe047147722cc97db0229e3549b29ccd9 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Fri, 19 Jan 2024 13:00:17 +0000 Subject: [PATCH 04/18] v0 --- src/py/flwr/common/recordset_utils.py | 177 +++++++++++++++++++++++--- 1 file changed, 162 insertions(+), 15 deletions(-) diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_utils.py index 3f6dfca09ffb..fa6a7d2e93c4 100644 --- a/src/py/flwr/common/recordset_utils.py +++ b/src/py/flwr/common/recordset_utils.py @@ -15,12 +15,14 @@ """RecordSet utilities.""" -from typing import OrderedDict, Dict +from typing import Dict, OrderedDict, Tuple, Union, cast, get_args from .configsrecord import ConfigsRecord +from .metricsrecord import MetricsRecord from .parametersrecord import Array, ParametersRecord from .recordset import RecordSet from .typing import ( + ConfigsRecordValues, EvaluateIns, EvaluateRes, FitIns, @@ -29,8 +31,10 @@ GetParametersRes, GetPropertiesIns, GetPropertiesRes, + MetricsRecordValues, Parameters, - ConfigsRecordValues + Scalar, + Status, ) @@ -100,41 +104,184 @@ def parameters_to_parametersrecord( return p_record -def fuse_configsrecord_data(configs: ConfigsRecord) -> Dict[str, ConfigsRecordValues]: - """Fuse all config entries into a single dictionary.""" - +def _check_mapping_from_scalar_to_metricsrecordstypes( + scalar_config: Dict[str, Scalar], +) -> Dict[str, MetricsRecordValues]: + """.""" + for value in scalar_config.values(): + if not isinstance(value, get_args(MetricsRecordValues)): + raise TypeError( + f"Supported types are {MetricsRecordValues}. " + f"But you used type: {type(value)}" + ) + return cast(Dict[str, MetricsRecordValues], scalar_config) + + +def _check_mapping_from_scalar_to_configsrecordstypes( + scalar_config: Dict[str, Scalar], +) -> Dict[str, ConfigsRecordValues]: + """.""" + for value in scalar_config.values(): + if not isinstance(value, get_args(ConfigsRecordValues)): + raise TypeError( + f"Supported types are {ConfigsRecordValues}. " + f"But you used type: {type(value)}" + ) + return cast(Dict[str, ConfigsRecordValues], scalar_config) + + +def _check_mapping_from_recordscalartype_to_scalar( + record_data: Dict[str, Union[ConfigsRecordValues, MetricsRecordValues]] +) -> Dict[str, Scalar]: + """Check mapping `common.*RecordValues` into `common.Scalar` is possible.""" + for value in record_data.values(): + if not isinstance(value, get_args(Scalar)): + raise TypeError( + "There is not a 1:1 mapping between `common.Scalar` types and those " + "supported in `common.ConfigsRecordValues` or " + "`common.ConfigsRecordValues`. Consider casting your values to a type " + "supported by the `common.RecordSet` infrastructure. " + f"You used type: {type(value)}" + ) + return cast(Dict[str, Scalar], record_data) -def recordset_to_fit_ins(recordset: RecordSet) -> FitIns: +def _recordset_to_fit_or_evaluate_ins( + recordset: RecordSet, ins_str: str +) -> Tuple[Parameters, Dict[str, Scalar]]: + """Derive Fit/Evaluate Ins from a RecordSet.""" + # get Array and construct Parameters + parameters_record = recordset.get_parameters(f"{ins_str}.parameters") + + parameters = parametersrecord_to_parameters(parameters_record) + + # get config dict + config_record = recordset.get_configs(f"{ins_str}.config") + + config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record.data) - config = recordset.configs + return parameters, config_dict - return FitIns() + +def _embed_status_into_recordset( + res_str: str, status: Status, recordset: RecordSet +) -> RecordSet: + status_dict: Dict[str, ConfigsRecordValues] = { + "code": status.code.value, + "message": status.message, + } + recordset.set_configs(f"{res_str}.status", record=ConfigsRecord(status_dict)) + return recordset + + +def recordset_to_fit_ins(recordset: RecordSet) -> FitIns: + """Derive FitIns from a RecordSet object.""" + parameters, config = _recordset_to_fit_or_evaluate_ins(recordset, ins_str="fitins") + + return FitIns(parameters=parameters, config=config) def fit_res_to_recordset(fitres: FitRes) -> RecordSet: - return RecordSet() + """Construct a RecordSet from a FitRes object.""" + recordset = RecordSet() + + metrics = _check_mapping_from_scalar_to_metricsrecordstypes(fitres.metrics) + recordset.set_metrics(name="fitres.metrics", record=MetricsRecord(metrics)) + recordset.set_metrics( + name="fitres.num_examples", + record=MetricsRecord({"num_examples": fitres.num_examples}), + ) + recordset.set_parameters( + name="fitres.parameters", + record=parameters_to_parametersrecord(fitres.parameters), + ) + + # status + recordset = _embed_status_into_recordset("fitres", fitres.status, recordset) + + return recordset def recodset_to_evaluate_ins(recordset: RecordSet) -> EvaluateIns: - return EvaluateIns() + """Derive EvaluateIns from a RecordSet object.""" + parameters, config = _recordset_to_fit_or_evaluate_ins( + recordset, ins_str="evaluateins" + ) + + return EvaluateIns(parameters=parameters, config=config) def evaluate_res_to_recordset(evaluateres: EvaluateRes) -> RecordSet: - return RecordSet() + """Construct a RecordSet from a EvaluateRes object.""" + recordset = RecordSet() + + # loss + recordset.set_metrics( + name="evaluateres.loss", + record=MetricsRecord({"loss": evaluateres.loss}), + ) + + # num_examples + recordset.set_metrics( + name="evaluateres.num_examples", + record=MetricsRecord({"num_examples": evaluateres.num_examples}), + ) + + # metrics + metrics = _check_mapping_from_scalar_to_metricsrecordstypes(evaluateres.metrics) + recordset.set_metrics(name="evaluateres.metrics", record=MetricsRecord(metrics)) + + # status + recordset = _embed_status_into_recordset( + "evaluateres", evaluateres.status, recordset + ) + + return recordset def recordset_to_getparameters_ins(recordset: RecordSet) -> GetParametersIns: - return GetParametersIns + """Derive GetParametersIns from a RecordSet object.""" + config_record = recordset.get_configs("getparametersins.config") + + config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record.data) + + return GetParametersIns(config=config_dict) def getparameters_res_to_recordset(getparametersres: GetParametersRes) -> RecordSet: - return RecordSet() + """Construct a RecordSet from a GetParametersRes object.""" + recordset = RecordSet() + parameters_record = parameters_to_parametersrecord(getparametersres.parameters) + recordset.set_parameters("getparametersres.parameters", parameters_record) + + # status + recordset = _embed_status_into_recordset( + "getparametersres", getparametersres.status, recordset + ) + + return recordset def recordset_to_getproperties_ins(recordset: RecordSet) -> GetPropertiesIns: - return GetPropertiesIns() + """Derive GetPropertiesIns from a RecordSet object.""" + config_record = recordset.get_configs("getpropertiesins.config") + config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record.data) + + return GetPropertiesIns(config=config_dict) def getproperties_res_to_recorset(getpropertiesres: GetPropertiesRes) -> RecordSet: - return RecordSet() + """Construct a RecordSet from a GetPropertiesRes object.""" + recordset = RecordSet() + configs = _check_mapping_from_scalar_to_configsrecordstypes( + getpropertiesres.properties + ) + recordset.set_configs( + name="gerpropertiesres.properties", record=ConfigsRecord(configs) + ) + # status + recordset = _embed_status_into_recordset( + "getpropertiesres", getpropertiesres.status, recordset + ) + + return recordset From c0dc2f1392e821b28b088c3640cbb19fa09d689c Mon Sep 17 00:00:00 2001 From: jafermarq Date: Fri, 19 Jan 2024 15:42:57 +0000 Subject: [PATCH 05/18] wip --- src/py/flwr/common/recordset_test.py | 61 ++++++++++++++- src/py/flwr/common/recordset_utils.py | 103 +++++++++++++++++++++----- 2 files changed, 144 insertions(+), 20 deletions(-) diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index 3f0917d75cf5..6de2f14535c6 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -14,7 +14,7 @@ # ============================================================================== """RecordSet tests.""" - +from functools import partial from typing import Callable, Dict, List, OrderedDict, Type, Union import numpy as np @@ -24,12 +24,19 @@ from .metricsrecord import MetricsRecord from .parameter import ndarrays_to_parameters, parameters_to_ndarrays from .parametersrecord import Array, ParametersRecord +from .recordset import RecordSet from .recordset_utils import ( + evaluate_ins_to_recordset, + fit_ins_to_recordset, parameters_to_parametersrecord, parametersrecord_to_parameters, + recordset_to_evaluate_ins, + recordset_to_fit_ins, ) from .typing import ( ConfigsRecordValues, + EvaluateIns, + FitIns, MetricsRecordValues, NDArray, NDArrays, @@ -333,3 +340,55 @@ def test_set_configs_to_configsrecord_with_incorrect_types( with pytest.raises(TypeError): m_record.set_configs(my_metrics) # type: ignore + + +def _get_recordset_compatible_with_legacy_ins(ins_str: str) -> RecordSet: + recordset = RecordSet() + + # add a ParametersRecord + array_dict = OrderedDict( + {str(i): ndarray_to_array(ndarray) for i, ndarray in enumerate(get_ndarrays())} + ) + recordset.set_parameters( + f"{ins_str}.parameters", record=ParametersRecord(array_dict) + ) + + # add a ConfigsRecord + recordset.set_configs( + f"{ins_str}.config", + record=ConfigsRecord({"a": 1, "b": 2.0, "c": np.eye(2).flatten().tobytes()}), + ) + + return recordset + + +@pytest.mark.parametrize( + "ins_str, do_func, undo_func", + [ + ( + "fitins", + partial(recordset_to_fit_ins, keep_input=True), + fit_ins_to_recordset, + ), + ( + "evaluateins", + partial(recordset_to_evaluate_ins, keep_input=True), + evaluate_ins_to_recordset, + ), + ], +) +def test_recordset_to_fit_or_evaluate_ins( + ins_str: str, + do_func: Callable[[RecordSet], Union[FitIns, EvaluateIns]], + undo_func: Callable[[Union[FitIns, EvaluateIns]], RecordSet], +) -> None: + """.""" + valid_record_set = _get_recordset_compatible_with_legacy_ins(ins_str) + + ins = do_func(valid_record_set) + + reverted_record_set = undo_func(ins) + + assert valid_record_set.configs == reverted_record_set.configs + # TODO: how to check parameters consistency (given than Array->Parameters is + # a destructive process ? (i.e. different metadata encoded)) diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_utils.py index fa6a7d2e93c4..9501a3605a5e 100644 --- a/src/py/flwr/common/recordset_utils.py +++ b/src/py/flwr/common/recordset_utils.py @@ -15,13 +15,14 @@ """RecordSet utilities.""" -from typing import Dict, OrderedDict, Tuple, Union, cast, get_args +from typing import Dict, Mapping, OrderedDict, Tuple, Union, cast, get_args from .configsrecord import ConfigsRecord from .metricsrecord import MetricsRecord from .parametersrecord import Array, ParametersRecord from .recordset import RecordSet from .typing import ( + Code, ConfigsRecordValues, EvaluateIns, EvaluateRes, @@ -131,7 +132,7 @@ def _check_mapping_from_scalar_to_configsrecordstypes( def _check_mapping_from_recordscalartype_to_scalar( - record_data: Dict[str, Union[ConfigsRecordValues, MetricsRecordValues]] + record_data: Mapping[str, Union[ConfigsRecordValues, MetricsRecordValues]] ) -> Dict[str, Scalar]: """Check mapping `common.*RecordValues` into `common.Scalar` is possible.""" for value in record_data.values(): @@ -146,14 +147,18 @@ def _check_mapping_from_recordscalartype_to_scalar( return cast(Dict[str, Scalar], record_data) -def _recordset_to_fit_or_evaluate_ins( - recordset: RecordSet, ins_str: str +def _recordset_to_fit_or_evaluate_ins_components( + recordset: RecordSet, + ins_str: str, + keep_input: bool, ) -> Tuple[Parameters, Dict[str, Scalar]]: """Derive Fit/Evaluate Ins from a RecordSet.""" # get Array and construct Parameters parameters_record = recordset.get_parameters(f"{ins_str}.parameters") - parameters = parametersrecord_to_parameters(parameters_record) + parameters = parametersrecord_to_parameters( + parameters_record, keep_input=keep_input + ) # get config dict config_record = recordset.get_configs(f"{ins_str}.config") @@ -163,77 +168,137 @@ def _recordset_to_fit_or_evaluate_ins( return parameters, config_dict +def _fit_or_evaluate_ins_to_recordset(ins: Union[FitIns, EvaluateIns]) -> RecordSet: + recordset = RecordSet() + + ins_str = "fitins" if isinstance(ins, FitIns) else "evaluateins" + recordset.set_parameters( + name=f"{ins_str}.parameters", + record=parameters_to_parametersrecord(ins.parameters), + ) + + config = _check_mapping_from_scalar_to_configsrecordstypes(ins.config) + recordset.set_configs(name=f"{ins_str}.config", record=ConfigsRecord(config)) + + return recordset + + def _embed_status_into_recordset( res_str: str, status: Status, recordset: RecordSet ) -> RecordSet: status_dict: Dict[str, ConfigsRecordValues] = { - "code": status.code.value, + "code": int(status.code.value), "message": status.message, } recordset.set_configs(f"{res_str}.status", record=ConfigsRecord(status_dict)) return recordset -def recordset_to_fit_ins(recordset: RecordSet) -> FitIns: +def _extract_status_from_recordset(res_str: str, recordset: RecordSet) -> Status: + status = recordset.get_metrics(f"{res_str}.status") + code = cast(int, status.data["code"]) + return Status(code=Code(code), message=str(status.data["message"])) + + +def recordset_to_fit_ins(recordset: RecordSet, keep_input: bool) -> FitIns: """Derive FitIns from a RecordSet object.""" - parameters, config = _recordset_to_fit_or_evaluate_ins(recordset, ins_str="fitins") + parameters, config = _recordset_to_fit_or_evaluate_ins_components( + recordset, + ins_str="fitins", + keep_input=keep_input, + ) return FitIns(parameters=parameters, config=config) +def fit_ins_to_recordset(fitins: FitIns) -> RecordSet: + """Construct a RecordSet from a FitIns object.""" + return _fit_or_evaluate_ins_to_recordset(fitins) + + +def recordset_to_fit_res(recordset: RecordSet) -> FitRes: + """Derive FitRes from a RecordSet object.""" + ins_str = "fitres" + parameters = parametersrecord_to_parameters( + recordset.get_parameters(f"{ins_str}.parameters") + ) + + num_examples = cast( + int, recordset.get_metrics(f"{ins_str}.num_examples").data["num_exampes"] + ) + metrics_record = recordset.get_metrics(f"{ins_str}.metrics") + + metrics = _check_mapping_from_recordscalartype_to_scalar(metrics_record.data) + status = _extract_status_from_recordset(f"{ins_str}", recordset) + + return FitRes( + status=status, parameters=parameters, num_examples=num_examples, metrics=metrics + ) + + def fit_res_to_recordset(fitres: FitRes) -> RecordSet: """Construct a RecordSet from a FitRes object.""" recordset = RecordSet() + res_str = "fitres" + metrics = _check_mapping_from_scalar_to_metricsrecordstypes(fitres.metrics) - recordset.set_metrics(name="fitres.metrics", record=MetricsRecord(metrics)) + recordset.set_metrics(name=f"{res_str}.metrics", record=MetricsRecord(metrics)) recordset.set_metrics( - name="fitres.num_examples", + name=f"{res_str}.num_examples", record=MetricsRecord({"num_examples": fitres.num_examples}), ) recordset.set_parameters( - name="fitres.parameters", + name=f"{res_str}.parameters", record=parameters_to_parametersrecord(fitres.parameters), ) # status - recordset = _embed_status_into_recordset("fitres", fitres.status, recordset) + recordset = _embed_status_into_recordset(f"{res_str}", fitres.status, recordset) return recordset -def recodset_to_evaluate_ins(recordset: RecordSet) -> EvaluateIns: +def recordset_to_evaluate_ins(recordset: RecordSet, keep_input: bool) -> EvaluateIns: """Derive EvaluateIns from a RecordSet object.""" - parameters, config = _recordset_to_fit_or_evaluate_ins( - recordset, ins_str="evaluateins" + parameters, config = _recordset_to_fit_or_evaluate_ins_components( + recordset, + ins_str="evaluateins", + keep_input=keep_input, ) return EvaluateIns(parameters=parameters, config=config) +def evaluate_ins_to_recordset(evaluateins: EvaluateIns) -> RecordSet: + """Construct a RecordSet from a EvaluateIns object.""" + return _fit_or_evaluate_ins_to_recordset(evaluateins) + + def evaluate_res_to_recordset(evaluateres: EvaluateRes) -> RecordSet: """Construct a RecordSet from a EvaluateRes object.""" recordset = RecordSet() + res_str = "evaluateres" # loss recordset.set_metrics( - name="evaluateres.loss", + name=f"{res_str}.loss", record=MetricsRecord({"loss": evaluateres.loss}), ) # num_examples recordset.set_metrics( - name="evaluateres.num_examples", + name=f"{res_str}.num_examples", record=MetricsRecord({"num_examples": evaluateres.num_examples}), ) # metrics metrics = _check_mapping_from_scalar_to_metricsrecordstypes(evaluateres.metrics) - recordset.set_metrics(name="evaluateres.metrics", record=MetricsRecord(metrics)) + recordset.set_metrics(name=f"{res_str}.metrics", record=MetricsRecord(metrics)) # status recordset = _embed_status_into_recordset( - "evaluateres", evaluateres.status, recordset + f"{res_str}", evaluateres.status, recordset ) return recordset From 3a2ed1c8a0af7c3558500a5a16266f666b9ecb51 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Fri, 19 Jan 2024 17:12:17 +0000 Subject: [PATCH 06/18] all conversions --- src/py/flwr/common/recordset_utils.py | 70 +++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_utils.py index 9501a3605a5e..00084ec5cc15 100644 --- a/src/py/flwr/common/recordset_utils.py +++ b/src/py/flwr/common/recordset_utils.py @@ -229,7 +229,7 @@ def recordset_to_fit_res(recordset: RecordSet) -> FitRes: metrics_record = recordset.get_metrics(f"{ins_str}.metrics") metrics = _check_mapping_from_recordscalartype_to_scalar(metrics_record.data) - status = _extract_status_from_recordset(f"{ins_str}", recordset) + status = _extract_status_from_recordset(ins_str, recordset) return FitRes( status=status, parameters=parameters, num_examples=num_examples, metrics=metrics @@ -254,7 +254,7 @@ def fit_res_to_recordset(fitres: FitRes) -> RecordSet: ) # status - recordset = _embed_status_into_recordset(f"{res_str}", fitres.status, recordset) + recordset = _embed_status_into_recordset(res_str, fitres.status, recordset) return recordset @@ -275,6 +275,25 @@ def evaluate_ins_to_recordset(evaluateins: EvaluateIns) -> RecordSet: return _fit_or_evaluate_ins_to_recordset(evaluateins) +def recordset_to_evaluate_res(recordset: RecordSet) -> EvaluateRes: + """Derive EvaluateRes from a RecordSet object.""" + ins_str = "evaluateres" + + loss = cast(int, recordset.get_metrics(f"{ins_str}.loss").data["loss"]) + + num_examples = cast( + int, recordset.get_metrics(f"{ins_str}.num_examples").data["num_exampes"] + ) + metrics_record = recordset.get_metrics(f"{ins_str}.metrics") + + metrics = _check_mapping_from_recordscalartype_to_scalar(metrics_record.data) + status = _extract_status_from_recordset(ins_str, recordset) + + return EvaluateRes( + status=status, loss=loss, num_examples=num_examples, metrics=metrics + ) + + def evaluate_res_to_recordset(evaluateres: EvaluateRes) -> RecordSet: """Construct a RecordSet from a EvaluateRes object.""" recordset = RecordSet() @@ -313,6 +332,15 @@ def recordset_to_getparameters_ins(recordset: RecordSet) -> GetParametersIns: return GetParametersIns(config=config_dict) +def getparameters_ins_to_recordset(getparameters_ins: GetParametersIns) -> RecordSet: + """Construct a RecordSet from a GetParametersIns object.""" + recordset = RecordSet() + + config = _check_mapping_from_scalar_to_configsrecordstypes(getparameters_ins.config) + recordset.set_configs(name="getparametersins.config", record=ConfigsRecord(config)) + return recordset + + def getparameters_res_to_recordset(getparametersres: GetParametersRes) -> RecordSet: """Construct a RecordSet from a GetParametersRes object.""" recordset = RecordSet() @@ -327,6 +355,17 @@ def getparameters_res_to_recordset(getparametersres: GetParametersRes) -> Record return recordset +def recordset_to_getparameters_res(recordset: RecordSet) -> GetParametersRes: + """Derive GetParametersRes from a RecordSet object.""" + res_str = "getparametersres" + parameters = parametersrecord_to_parameters( + recordset.get_parameters(f"{res_str}.parameters") + ) + + status = _extract_status_from_recordset(res_str, recordset) + return GetParametersRes(status=status, parameters=parameters) + + def recordset_to_getproperties_ins(recordset: RecordSet) -> GetPropertiesIns: """Derive GetPropertiesIns from a RecordSet object.""" config_record = recordset.get_configs("getpropertiesins.config") @@ -335,14 +374,37 @@ def recordset_to_getproperties_ins(recordset: RecordSet) -> GetPropertiesIns: return GetPropertiesIns(config=config_dict) -def getproperties_res_to_recorset(getpropertiesres: GetPropertiesRes) -> RecordSet: +def getproperties_ins_to_recordset(getpropertiesins: GetPropertiesIns) -> RecordSet: + """Construct a RecordSet from a GetPropertiesRes object.""" + recordset = RecordSet() + config_dict = _check_mapping_from_scalar_to_configsrecordstypes( + getpropertiesins.config + ) + recordset.set_configs( + name="getpropertiesins.config", record=ConfigsRecord(config_dict) + ) + return recordset + + +def recordset_to_getproperties_res(recordset: RecordSet) -> GetPropertiesRes: + """Derive GetPropertiesRes from a RecordSet object.""" + res_str = "getpropertiesres" + config_record = recordset.get_configs(f"{res_str}.config") + properties = _check_mapping_from_recordscalartype_to_scalar(config_record.data) + + status = _extract_status_from_recordset(res_str, recordset=recordset) + + return GetPropertiesRes(status=status, properties=properties) + + +def getproperties_res_to_recordset(getpropertiesres: GetPropertiesRes) -> RecordSet: """Construct a RecordSet from a GetPropertiesRes object.""" recordset = RecordSet() configs = _check_mapping_from_scalar_to_configsrecordstypes( getpropertiesres.properties ) recordset.set_configs( - name="gerpropertiesres.properties", record=ConfigsRecord(configs) + name="getpropertiesres.properties", record=ConfigsRecord(configs) ) # status recordset = _embed_status_into_recordset( From 123228a657c48b54ec7db1856a0d88acf13a29bc Mon Sep 17 00:00:00 2001 From: jafermarq Date: Fri, 19 Jan 2024 19:33:36 +0000 Subject: [PATCH 07/18] wip --- src/py/flwr/common/recordset_test.py | 118 +++++++++++++++++++++++++- src/py/flwr/common/recordset_utils.py | 33 ++++--- 2 files changed, 135 insertions(+), 16 deletions(-) diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index 2fe09c703807..5119d2b2128c 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -14,8 +14,10 @@ # ============================================================================== """RecordSet tests.""" +from contextlib import nullcontext +from copy import deepcopy from functools import partial -from typing import Callable, Dict, List, OrderedDict, Type, Union +from typing import Callable, Dict, List, OrderedDict, Type, Union, Any import numpy as np import pytest @@ -26,23 +28,38 @@ from .parametersrecord import Array, ParametersRecord from .recordset import RecordSet from .recordset_utils import ( + _embed_status_into_recordset, evaluate_ins_to_recordset, fit_ins_to_recordset, + getproperties_ins_to_recordset, + getproperties_res_to_recordset, parameters_to_parametersrecord, parametersrecord_to_parameters, recordset_to_evaluate_ins, recordset_to_fit_ins, + recordset_to_getproperties_ins, + recordset_to_getproperties_res, + evaluate_res_to_recordset, + recordset_to_evaluate_res, ) from .typing import ( + Code, + Scalar, ConfigsRecordValues, EvaluateIns, FitIns, + GetPropertiesIns, + EvaluateRes, + GetPropertiesRes, MetricsRecordValues, NDArray, NDArrays, Parameters, + Status, ) +from flwr.client.message_handler.message_handler_test import ClientWithProps, _get_client_fn + def get_ndarrays() -> NDArrays: """Return list of NumPy arrays.""" @@ -342,6 +359,36 @@ def test_set_configs_to_configsrecord_with_incorrect_types( m_record.set_configs(my_metrics) # type: ignore + +@pytest.mark.parametrize( + "context, config", + [ + (nullcontext(), {'a': 1.0, 'b': 0}), + (pytest.raises(TypeError), {'a': 1.0, 'b': 3, 'c': True}), # fails due to unsupported type for configrecord value + ], +) +def test_fitins_to_recordset_and_back(context: Any, config: Dict[str, Scalar]) -> None: + + arrays = get_ndarrays() + fitins = FitIns(parameters=ndarrays_to_parameters(arrays), config=config) + + fitins_copy = deepcopy(fitins) + + with context: + recordset = fit_ins_to_recordset(fitins) + + fitins_ = recordset_to_fit_ins(recordset, keep_input=False) + + + assert fitins_copy == fitins_ + + + +###################### DELETE FROM BELOW ################################# +###################### DELETE FROM BELOW ################################# +###################### DELETE FROM BELOW ################################# +###################### DELETE FROM BELOW ################################# + def _get_recordset_compatible_with_legacy_ins(ins_str: str) -> RecordSet: recordset = RecordSet() @@ -377,7 +424,7 @@ def _get_recordset_compatible_with_legacy_ins(ins_str: str) -> RecordSet: ), ], ) -def test_recordset_to_fit_or_evaluate_ins( +def test_recordset_to_fit_or_evaluate_ins_and_back( ins_str: str, do_func: Callable[[RecordSet], Union[FitIns, EvaluateIns]], undo_func: Callable[[Union[FitIns, EvaluateIns]], RecordSet], @@ -392,3 +439,70 @@ def test_recordset_to_fit_or_evaluate_ins( assert valid_record_set.configs == reverted_record_set.configs # TODO: how to check parameters consistency (given than Array->Parameters is # a destructive process ? (i.e. different metadata encoded)) + + + +@pytest.mark.parametrize( + "ins_str, do_func, undo_func", + [ + ( + "getevaluateres", + recordset_to_evaluate_res, + evaluate_res_to_recordset, + ), + ], +) +def test_recordset_to_evaluate_res_and_back( + ins_str: str, + do_func: Callable[[RecordSet], EvaluateRes], + undo_func: Callable[[EvaluateRes], RecordSet], +) -> None: + + recordset = RecordSet() + + + +def test_getproperties_res_to_recordset_and_back() -> None: + """.""" + client_fn = _get_client_fn(ClientWithProps()) + + +@pytest.mark.parametrize( + "ins_str, do_func, undo_func", + [ + ( + "getpropertiesins", + recordset_to_getproperties_ins, + getproperties_ins_to_recordset, + ), + ( + "getpropertiesres", + recordset_to_getproperties_res, + getproperties_res_to_recordset, + ), + ], +) +def test_recordset_to_get_properties_ins_or_res_and_back( + ins_str: str, + do_func: Callable[[RecordSet], Union[GetPropertiesIns, GetPropertiesRes]], + undo_func: Callable[[Union[GetPropertiesIns, GetPropertiesRes]], RecordSet], +) -> None: + """.""" + recordset = RecordSet() + recordset.set_configs( + f"{ins_str}.{'properties' if 'res' in ins_str else 'config'}", + record=ConfigsRecord({"a": 1, "b": 2.0, "c": np.eye(2).flatten().tobytes()}), + ) + + # embed status if it's a response message only + if "res" in ins_str: + recordset = _embed_status_into_recordset( + ins_str, status=Status(code=Code(0), message="hello"), recordset=recordset + ) + + recordset_copy = deepcopy(recordset) + + ins = do_func(recordset) + + recordset_ = undo_func(ins) + assert recordset_copy == recordset_ diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_utils.py index 00084ec5cc15..d5f41e76f567 100644 --- a/src/py/flwr/common/recordset_utils.py +++ b/src/py/flwr/common/recordset_utils.py @@ -60,7 +60,12 @@ def parametersrecord_to_parameters( parameters = Parameters(tensors=[], tensor_type="") for key in list(record.data.keys()): - parameters.tensors.append(record.data[key].data) + parameters.tensors.append(record[key].data) + + if not parameters.tensor_type: + # Setting from first array in record. Recall the warning in the docstrings + # of this function. + parameters.tensor_type = record[key].stype if not keep_input: del record.data[key] @@ -190,14 +195,14 @@ def _embed_status_into_recordset( "code": int(status.code.value), "message": status.message, } - recordset.set_configs(f"{res_str}.status", record=ConfigsRecord(status_dict)) + recordset.set_metrics(f"{res_str}.status", record=ConfigsRecord(status_dict)) return recordset def _extract_status_from_recordset(res_str: str, recordset: RecordSet) -> Status: status = recordset.get_metrics(f"{res_str}.status") - code = cast(int, status.data["code"]) - return Status(code=Code(code), message=str(status.data["message"])) + code = cast(int, status["code"]) + return Status(code=Code(code), message=str(status["message"])) def recordset_to_fit_ins(recordset: RecordSet, keep_input: bool) -> FitIns: @@ -224,7 +229,7 @@ def recordset_to_fit_res(recordset: RecordSet) -> FitRes: ) num_examples = cast( - int, recordset.get_metrics(f"{ins_str}.num_examples").data["num_exampes"] + int, recordset.get_metrics(f"{ins_str}.num_examples")["num_exampes"] ) metrics_record = recordset.get_metrics(f"{ins_str}.metrics") @@ -279,10 +284,10 @@ def recordset_to_evaluate_res(recordset: RecordSet) -> EvaluateRes: """Derive EvaluateRes from a RecordSet object.""" ins_str = "evaluateres" - loss = cast(int, recordset.get_metrics(f"{ins_str}.loss").data["loss"]) + loss = cast(int, recordset.get_metrics(f"{ins_str}.loss")["loss"]) num_examples = cast( - int, recordset.get_metrics(f"{ins_str}.num_examples").data["num_exampes"] + int, recordset.get_metrics(f"{ins_str}.num_examples")["num_exampes"] ) metrics_record = recordset.get_metrics(f"{ins_str}.metrics") @@ -344,12 +349,13 @@ def getparameters_ins_to_recordset(getparameters_ins: GetParametersIns) -> Recor def getparameters_res_to_recordset(getparametersres: GetParametersRes) -> RecordSet: """Construct a RecordSet from a GetParametersRes object.""" recordset = RecordSet() + res_str = "getparametersres" parameters_record = parameters_to_parametersrecord(getparametersres.parameters) - recordset.set_parameters("getparametersres.parameters", parameters_record) + recordset.set_parameters(f"{res_str}.parameters", parameters_record) # status recordset = _embed_status_into_recordset( - "getparametersres", getparametersres.status, recordset + res_str, getparametersres.status, recordset ) return recordset @@ -389,7 +395,7 @@ def getproperties_ins_to_recordset(getpropertiesins: GetPropertiesIns) -> Record def recordset_to_getproperties_res(recordset: RecordSet) -> GetPropertiesRes: """Derive GetPropertiesRes from a RecordSet object.""" res_str = "getpropertiesres" - config_record = recordset.get_configs(f"{res_str}.config") + config_record = recordset.get_configs(f"{res_str}.properties") properties = _check_mapping_from_recordscalartype_to_scalar(config_record.data) status = _extract_status_from_recordset(res_str, recordset=recordset) @@ -400,15 +406,14 @@ def recordset_to_getproperties_res(recordset: RecordSet) -> GetPropertiesRes: def getproperties_res_to_recordset(getpropertiesres: GetPropertiesRes) -> RecordSet: """Construct a RecordSet from a GetPropertiesRes object.""" recordset = RecordSet() + res_str = "getpropertiesres" configs = _check_mapping_from_scalar_to_configsrecordstypes( getpropertiesres.properties ) - recordset.set_configs( - name="getpropertiesres.properties", record=ConfigsRecord(configs) - ) + recordset.set_configs(name=f"{res_str}.properties", record=ConfigsRecord(configs)) # status recordset = _embed_status_into_recordset( - "getpropertiesres", getpropertiesres.status, recordset + res_str, getpropertiesres.status, recordset ) return recordset From cbb9766a87c1ef9ee488d6fad40fd828e790d594 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Fri, 19 Jan 2024 21:22:20 +0000 Subject: [PATCH 08/18] v1; ++tests --- src/py/flwr/common/recordset_test.py | 262 +++++++++++++++----------- src/py/flwr/common/recordset_utils.py | 34 ++-- 2 files changed, 166 insertions(+), 130 deletions(-) diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index 091f83037ef6..2b78ba20ea41 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -16,8 +16,7 @@ from contextlib import nullcontext from copy import deepcopy -from functools import partial -from typing import Callable, Dict, List, OrderedDict, Type, Union, Any +from typing import Any, Callable, Dict, List, OrderedDict, Type, Union import numpy as np import pytest @@ -26,40 +25,45 @@ from .metricsrecord import MetricsRecord from .parameter import ndarrays_to_parameters, parameters_to_ndarrays from .parametersrecord import Array, ParametersRecord -from .recordset import RecordSet from .recordset_utils import ( - _embed_status_into_recordset, evaluate_ins_to_recordset, + evaluate_res_to_recordset, fit_ins_to_recordset, + fit_res_to_recordset, + getparameters_ins_to_recordset, + getparameters_res_to_recordset, getproperties_ins_to_recordset, getproperties_res_to_recordset, parameters_to_parametersrecord, parametersrecord_to_parameters, recordset_to_evaluate_ins, + recordset_to_evaluate_res, recordset_to_fit_ins, + recordset_to_fit_res, + recordset_to_getparameters_ins, + recordset_to_getparameters_res, recordset_to_getproperties_ins, recordset_to_getproperties_res, - evaluate_res_to_recordset, - recordset_to_evaluate_res, ) from .typing import ( Code, - Scalar, ConfigsRecordValues, EvaluateIns, + EvaluateRes, FitIns, + FitRes, + GetParametersIns, + GetParametersRes, GetPropertiesIns, - EvaluateRes, GetPropertiesRes, MetricsRecordValues, NDArray, NDArrays, Parameters, + Scalar, Status, ) -from flwr.client.message_handler.message_handler_test import ClientWithProps, _get_client_fn - def get_ndarrays() -> NDArrays: """Return list of NumPy arrays.""" @@ -362,150 +366,178 @@ def test_set_configs_to_configsrecord_with_incorrect_types( m_record.set_configs(my_metrics) # type: ignore +################################################## +# Testing conversion: *Ins --> RecordSet --> *Ins +# Testing conversion: *Res <-- RecordSet <-- *Res +################################################## -@pytest.mark.parametrize( - "context, config", - [ - (nullcontext(), {'a': 1.0, 'b': 0}), - (pytest.raises(TypeError), {'a': 1.0, 'b': 3, 'c': True}), # fails due to unsupported type for configrecord value - ], -) -def test_fitins_to_recordset_and_back(context: Any, config: Dict[str, Scalar]) -> None: +def test_fitins_to_recordset_and_back() -> None: + """Test conversion FitIns --> RecordSet --> FitIns.""" arrays = get_ndarrays() - fitins = FitIns(parameters=ndarrays_to_parameters(arrays), config=config) + fitins = FitIns( + parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0} + ) fitins_copy = deepcopy(fitins) - with context: - recordset = fit_ins_to_recordset(fitins) - - fitins_ = recordset_to_fit_ins(recordset, keep_input=False) + recordset = fit_ins_to_recordset(fitins, keep_input=False) + fitins_ = recordset_to_fit_ins(recordset, keep_input=False) assert fitins_copy == fitins_ +@pytest.mark.parametrize( + "context, metrics", + [ + (nullcontext(), {"a": 1.0, "b": 0}), + ( + pytest.raises(TypeError), + {"a": 1.0, "b": 3, "c": True}, + ), # fails due to unsupported type for metricsrecord value + ], +) +def test_fitres_to_recordset_and_back(context: Any, metrics: Dict[str, Scalar]) -> None: + """Test conversion FitRes --> RecordSet --> FitRes.""" + arrays = get_ndarrays() + fitres = FitRes( + parameters=ndarrays_to_parameters(arrays), + num_examples=1, + status=Status(code=Code(0), message=""), + metrics=metrics, + ) -###################### DELETE FROM BELOW ################################# -###################### DELETE FROM BELOW ################################# -###################### DELETE FROM BELOW ################################# -###################### DELETE FROM BELOW ################################# + fitres_copy = deepcopy(fitres) -def _get_recordset_compatible_with_legacy_ins(ins_str: str) -> RecordSet: - recordset = RecordSet() + with context: + recordset = fit_res_to_recordset(fitres, keep_input=False) + fitres_ = recordset_to_fit_res(recordset, keep_input=False) - # add a ParametersRecord - array_dict = OrderedDict( - {str(i): ndarray_to_array(ndarray) for i, ndarray in enumerate(get_ndarrays())} - ) - recordset.set_parameters( - f"{ins_str}.parameters", record=ParametersRecord(array_dict) - ) + # only check if we didn't test for an invalid setting. Only in valid settings + # makes sense to evaluate the below, since both functions above have succesfully + # being executed. + if isinstance(context, nullcontext): + assert fitres_copy == fitres_ - # add a ConfigsRecord - recordset.set_configs( - f"{ins_str}.config", - record=ConfigsRecord({"a": 1, "b": 2.0, "c": np.eye(2).flatten().tobytes()}), + +def test_evaluateins_to_recordset_and_back() -> None: + """Test conversion EvaluateIns --> RecordSet --> EvaluateIns.""" + arrays = get_ndarrays() + evaluateins = EvaluateIns( + parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0} ) - return recordset + evaluateins_copy = deepcopy(evaluateins) + + recordset = evaluate_ins_to_recordset(evaluateins, keep_input=False) + + evaluateins_ = recordset_to_evaluate_ins(recordset, keep_input=False) + + assert evaluateins_copy == evaluateins_ @pytest.mark.parametrize( - "ins_str, do_func, undo_func", + "context, metrics", [ + (nullcontext(), {"a": 1.0, "b": 0}), ( - "fitins", - partial(recordset_to_fit_ins, keep_input=True), - fit_ins_to_recordset, - ), - ( - "evaluateins", - partial(recordset_to_evaluate_ins, keep_input=True), - evaluate_ins_to_recordset, - ), + pytest.raises(TypeError), + {"a": 1.0, "b": 3, "c": True}, + ), # fails due to unsupported type for metricsrecord value ], ) -def test_recordset_to_fit_or_evaluate_ins_and_back( - ins_str: str, - do_func: Callable[[RecordSet], Union[FitIns, EvaluateIns]], - undo_func: Callable[[Union[FitIns, EvaluateIns]], RecordSet], +def test_evaluateres_to_recordset_and_back( + context: Any, metrics: Dict[str, Scalar] ) -> None: - """.""" - valid_record_set = _get_recordset_compatible_with_legacy_ins(ins_str) + """Test conversion EvaluateRes --> RecordSet --> EvaluateRes.""" + evaluateres = EvaluateRes( + num_examples=1, + loss=0.1, + status=Status(code=Code(0), message=""), + metrics=metrics, + ) - ins = do_func(valid_record_set) + evaluateres_copy = deepcopy(evaluateres) - reverted_record_set = undo_func(ins) + with context: + recordset = evaluate_res_to_recordset(evaluateres) + evaluateres_ = recordset_to_evaluate_res(recordset) - assert valid_record_set.configs == reverted_record_set.configs - # TODO: how to check parameters consistency (given than Array->Parameters is - # a destructive process ? (i.e. different metadata encoded)) + # only check if we didn't test for an invalid setting. Only in valid settings + # makes sense to evaluate the below, since both functions above have succesfully + # being executed. + if isinstance(context, nullcontext): + assert evaluateres_copy == evaluateres_ +def test_get_properties_ins_to_recordset_and_back() -> None: + """Test conversion GetPropertiesIns --> RecordSet --> GetPropertiesIns.""" + config_dict: Dict[str, Scalar] = { + "a": 1.0, + "b": 3, + "c": True, + } # valid since both Ins/Res communicate over ConfigsRecord -@pytest.mark.parametrize( - "ins_str, do_func, undo_func", - [ - ( - "getevaluateres", - recordset_to_evaluate_res, - evaluate_res_to_recordset, - ), - ], -) -def test_recordset_to_evaluate_res_and_back( - ins_str: str, - do_func: Callable[[RecordSet], EvaluateRes], - undo_func: Callable[[EvaluateRes], RecordSet], -) -> None: + getproperties_ins = GetPropertiesIns(config_dict) - recordset = RecordSet() + getproperties_ins_copy = deepcopy(getproperties_ins) + recordset = getproperties_ins_to_recordset(getproperties_ins) + getproperties_ins_ = recordset_to_getproperties_ins(recordset) + assert getproperties_ins_copy == getproperties_ins_ -def test_getproperties_res_to_recordset_and_back() -> None: - """.""" - client_fn = _get_client_fn(ClientWithProps()) +def test_get_properties_res_to_recordset_and_back() -> None: + """Test conversion GetPropertiesRes --> RecordSet --> GetPropertiesRes.""" + config_dict: Dict[str, Scalar] = { + "a": 1.0, + "b": 3, + "c": True, + } # valid since both Ins/Res communicate over ConfigsRecord -@pytest.mark.parametrize( - "ins_str, do_func, undo_func", - [ - ( - "getpropertiesins", - recordset_to_getproperties_ins, - getproperties_ins_to_recordset, - ), - ( - "getpropertiesres", - recordset_to_getproperties_res, - getproperties_res_to_recordset, - ), - ], -) -def test_recordset_to_get_properties_ins_or_res_and_back( - ins_str: str, - do_func: Callable[[RecordSet], Union[GetPropertiesIns, GetPropertiesRes]], - undo_func: Callable[[Union[GetPropertiesIns, GetPropertiesRes]], RecordSet], -) -> None: - """.""" - recordset = RecordSet() - recordset.set_configs( - f"{ins_str}.{'properties' if 'res' in ins_str else 'config'}", - record=ConfigsRecord({"a": 1, "b": 2.0, "c": np.eye(2).flatten().tobytes()}), + getproperties_res = GetPropertiesRes( + status=Status(code=Code(0), message=""), properties=config_dict ) - # embed status if it's a response message only - if "res" in ins_str: - recordset = _embed_status_into_recordset( - ins_str, status=Status(code=Code(0), message="hello"), recordset=recordset - ) + getproperties_res_copy = deepcopy(getproperties_res) + + recordset = getproperties_res_to_recordset(getproperties_res) + getproperties_res_ = recordset_to_getproperties_res(recordset) + + assert getproperties_res_copy == getproperties_res_ + + +def test_get_parameters_ins_to_recordset_and_back() -> None: + """Test conversion GetParametersIns --> RecordSet --> GetParametersIns.""" + config_dict: Dict[str, Scalar] = { + "a": 1.0, + "b": 3, + "c": True, + } # valid since both Ins/Res communicate over ConfigsRecord + + getparameters_ins = GetParametersIns(config_dict) + + getparameters_ins_copy = deepcopy(getparameters_ins) + + recordset = getparameters_ins_to_recordset(getparameters_ins) + getparameters_ins_ = recordset_to_getparameters_ins(recordset) + + assert getparameters_ins_copy == getparameters_ins_ + + +def test_get_parameters_res_to_recordset_and_back() -> None: + """Test conversion GetParametersRes --> RecordSet --> GetParametersRes.""" + arrays = get_ndarrays() + getparameteres_res = GetParametersRes( + status=Status(code=Code(0), message=""), + parameters=ndarrays_to_parameters(arrays), + ) - recordset_copy = deepcopy(recordset) + getparameters_res_copy = deepcopy(getparameteres_res) - ins = do_func(recordset) + recordset = getparameters_res_to_recordset(getparameteres_res) + getparameteres_res_ = recordset_to_getparameters_res(recordset) - recordset_ = undo_func(ins) - assert recordset_copy == recordset_ + assert getparameters_res_copy == getparameteres_res_ diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_utils.py index d5f41e76f567..f5a347b677a2 100644 --- a/src/py/flwr/common/recordset_utils.py +++ b/src/py/flwr/common/recordset_utils.py @@ -64,7 +64,7 @@ def parametersrecord_to_parameters( if not parameters.tensor_type: # Setting from first array in record. Recall the warning in the docstrings - # of this function. + # of this function. parameters.tensor_type = record[key].stype if not keep_input: @@ -173,13 +173,15 @@ def _recordset_to_fit_or_evaluate_ins_components( return parameters, config_dict -def _fit_or_evaluate_ins_to_recordset(ins: Union[FitIns, EvaluateIns]) -> RecordSet: +def _fit_or_evaluate_ins_to_recordset( + ins: Union[FitIns, EvaluateIns], keep_input: bool +) -> RecordSet: recordset = RecordSet() ins_str = "fitins" if isinstance(ins, FitIns) else "evaluateins" recordset.set_parameters( name=f"{ins_str}.parameters", - record=parameters_to_parametersrecord(ins.parameters), + record=parameters_to_parametersrecord(ins.parameters, keep_input=keep_input), ) config = _check_mapping_from_scalar_to_configsrecordstypes(ins.config) @@ -195,12 +197,14 @@ def _embed_status_into_recordset( "code": int(status.code.value), "message": status.message, } - recordset.set_metrics(f"{res_str}.status", record=ConfigsRecord(status_dict)) + # we add it to a `ConfigsRecord`` because the `status.message`` is a string + # and `str` values aren't supported in `MetricsRecords` + recordset.set_configs(f"{res_str}.status", record=ConfigsRecord(status_dict)) return recordset def _extract_status_from_recordset(res_str: str, recordset: RecordSet) -> Status: - status = recordset.get_metrics(f"{res_str}.status") + status = recordset.get_configs(f"{res_str}.status") code = cast(int, status["code"]) return Status(code=Code(code), message=str(status["message"])) @@ -216,20 +220,20 @@ def recordset_to_fit_ins(recordset: RecordSet, keep_input: bool) -> FitIns: return FitIns(parameters=parameters, config=config) -def fit_ins_to_recordset(fitins: FitIns) -> RecordSet: +def fit_ins_to_recordset(fitins: FitIns, keep_input: bool) -> RecordSet: """Construct a RecordSet from a FitIns object.""" - return _fit_or_evaluate_ins_to_recordset(fitins) + return _fit_or_evaluate_ins_to_recordset(fitins, keep_input) -def recordset_to_fit_res(recordset: RecordSet) -> FitRes: +def recordset_to_fit_res(recordset: RecordSet, keep_input: bool) -> FitRes: """Derive FitRes from a RecordSet object.""" ins_str = "fitres" parameters = parametersrecord_to_parameters( - recordset.get_parameters(f"{ins_str}.parameters") + recordset.get_parameters(f"{ins_str}.parameters"), keep_input=keep_input ) num_examples = cast( - int, recordset.get_metrics(f"{ins_str}.num_examples")["num_exampes"] + int, recordset.get_metrics(f"{ins_str}.num_examples")["num_examples"] ) metrics_record = recordset.get_metrics(f"{ins_str}.metrics") @@ -241,7 +245,7 @@ def recordset_to_fit_res(recordset: RecordSet) -> FitRes: ) -def fit_res_to_recordset(fitres: FitRes) -> RecordSet: +def fit_res_to_recordset(fitres: FitRes, keep_input: bool) -> RecordSet: """Construct a RecordSet from a FitRes object.""" recordset = RecordSet() @@ -255,7 +259,7 @@ def fit_res_to_recordset(fitres: FitRes) -> RecordSet: ) recordset.set_parameters( name=f"{res_str}.parameters", - record=parameters_to_parametersrecord(fitres.parameters), + record=parameters_to_parametersrecord(fitres.parameters, keep_input), ) # status @@ -275,9 +279,9 @@ def recordset_to_evaluate_ins(recordset: RecordSet, keep_input: bool) -> Evaluat return EvaluateIns(parameters=parameters, config=config) -def evaluate_ins_to_recordset(evaluateins: EvaluateIns) -> RecordSet: +def evaluate_ins_to_recordset(evaluateins: EvaluateIns, keep_input: bool) -> RecordSet: """Construct a RecordSet from a EvaluateIns object.""" - return _fit_or_evaluate_ins_to_recordset(evaluateins) + return _fit_or_evaluate_ins_to_recordset(evaluateins, keep_input) def recordset_to_evaluate_res(recordset: RecordSet) -> EvaluateRes: @@ -287,7 +291,7 @@ def recordset_to_evaluate_res(recordset: RecordSet) -> EvaluateRes: loss = cast(int, recordset.get_metrics(f"{ins_str}.loss")["loss"]) num_examples = cast( - int, recordset.get_metrics(f"{ins_str}.num_examples")["num_exampes"] + int, recordset.get_metrics(f"{ins_str}.num_examples")["num_examples"] ) metrics_record = recordset.get_metrics(f"{ins_str}.metrics") From 20f70efc55d0980e72b7666ac1d436f7f3058cff Mon Sep 17 00:00:00 2001 From: jafermarq Date: Mon, 22 Jan 2024 11:17:55 +0000 Subject: [PATCH 09/18] more --- src/py/flwr/common/recordset_test.py | 100 +++++++++++++++++++++------ 1 file changed, 78 insertions(+), 22 deletions(-) diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index 2b78ba20ea41..3eb9e1f75ec5 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -16,15 +16,18 @@ from contextlib import nullcontext from copy import deepcopy +from functools import partial from typing import Any, Callable, Dict, List, OrderedDict, Type, Union import numpy as np import pytest from .configsrecord import ConfigsRecord +from .flowercontext import FlowerContext, Metadata from .metricsrecord import MetricsRecord from .parameter import ndarrays_to_parameters, parameters_to_ndarrays from .parametersrecord import Array, ParametersRecord +from .recordset import RecordSet from .recordset_utils import ( evaluate_ins_to_recordset, evaluate_res_to_recordset, @@ -372,12 +375,34 @@ def test_set_configs_to_configsrecord_with_incorrect_types( ################################################## +def _get_valid_fitins() -> FitIns: + arrays = get_ndarrays() + return FitIns(parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0}) + + +def _get_valid_evaluateins() -> EvaluateIns: + fit_ins = _get_valid_fitins() + return EvaluateIns(parameters=fit_ins.parameters, config=fit_ins.config) + + +def _get_valid_getparametersins() -> GetParametersIns: + config_dict: Dict[str, Scalar] = { + "a": 1.0, + "b": 3, + "c": True, + } # valid since both Ins/Res communicate over ConfigsRecord + + return GetParametersIns(config_dict) + + +def _get_valid_getpropertiesins() -> GetPropertiesIns: + getparamsins = _get_valid_getparametersins() + return GetPropertiesIns(config=getparamsins.config) + + def test_fitins_to_recordset_and_back() -> None: """Test conversion FitIns --> RecordSet --> FitIns.""" - arrays = get_ndarrays() - fitins = FitIns( - parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0} - ) + fitins = _get_valid_fitins() fitins_copy = deepcopy(fitins) @@ -423,10 +448,7 @@ def test_fitres_to_recordset_and_back(context: Any, metrics: Dict[str, Scalar]) def test_evaluateins_to_recordset_and_back() -> None: """Test conversion EvaluateIns --> RecordSet --> EvaluateIns.""" - arrays = get_ndarrays() - evaluateins = EvaluateIns( - parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0} - ) + evaluateins = _get_valid_evaluateins() evaluateins_copy = deepcopy(evaluateins) @@ -473,13 +495,7 @@ def test_evaluateres_to_recordset_and_back( def test_get_properties_ins_to_recordset_and_back() -> None: """Test conversion GetPropertiesIns --> RecordSet --> GetPropertiesIns.""" - config_dict: Dict[str, Scalar] = { - "a": 1.0, - "b": 3, - "c": True, - } # valid since both Ins/Res communicate over ConfigsRecord - - getproperties_ins = GetPropertiesIns(config_dict) + getproperties_ins = _get_valid_getpropertiesins() getproperties_ins_copy = deepcopy(getproperties_ins) @@ -511,13 +527,7 @@ def test_get_properties_res_to_recordset_and_back() -> None: def test_get_parameters_ins_to_recordset_and_back() -> None: """Test conversion GetParametersIns --> RecordSet --> GetParametersIns.""" - config_dict: Dict[str, Scalar] = { - "a": 1.0, - "b": 3, - "c": True, - } # valid since both Ins/Res communicate over ConfigsRecord - - getparameters_ins = GetParametersIns(config_dict) + getparameters_ins = _get_valid_getparametersins() getparameters_ins_copy = deepcopy(getparameters_ins) @@ -541,3 +551,49 @@ def test_get_parameters_res_to_recordset_and_back() -> None: getparameteres_res_ = recordset_to_getparameters_res(recordset) assert getparameters_res_copy == getparameteres_res_ + + +@pytest.mark.parametrize( + "ins, convert_fn, task_type", + [ + (_get_valid_fitins, partial(fit_ins_to_recordset, keep_input=False), "fit_ins"), + ( + _get_valid_evaluateins, + partial(evaluate_ins_to_recordset, keep_input=False), + "evaluate_ins", + ), + ( + _get_valid_getpropertiesins, + getproperties_ins_to_recordset, + "get_properties_ins", + ), + ( + _get_valid_getparametersins, + getparameters_ins_to_recordset, + "get_parameters_ins", + ), + ], +) +def test_flowercontext_driver_to_client( + ins: Union[FitIns, EvaluateIns, GetPropertiesIns, GetParametersIns], + convert_fn: Union[ + Callable[[FitIns], RecordSet], + Callable[[EvaluateIns], RecordSet], + Callable[[GetPropertiesIns], RecordSet], + Callable[[GetParametersIns], RecordSet], + ], + task_type: str, +) -> None: + """.""" + f_context = FlowerContext( + in_message=RecordSet, + out_message=convert_fn(ins()), + local=RecordSet(), + metadata=Metadata( + run_id=0, task_id="", group_id="", ttl="", task_type=task_type + ), + ) + + # TODO: embedd `f_context` in TaskIns + + # Construct FlowerContext from TaskIns From 1c64ea696bf750ee3264a91552345945174a4b69 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Mon, 22 Jan 2024 11:24:07 +0000 Subject: [PATCH 10/18] . --- src/py/flwr/common/recordset_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index 3eb9e1f75ec5..a9c63e783e7c 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -586,7 +586,7 @@ def test_flowercontext_driver_to_client( ) -> None: """.""" f_context = FlowerContext( - in_message=RecordSet, + in_message=RecordSet(), out_message=convert_fn(ins()), local=RecordSet(), metadata=Metadata( From 3064a8e49463abac10c11966d55a274d2af0b6fe Mon Sep 17 00:00:00 2001 From: jafermarq Date: Mon, 22 Jan 2024 17:08:08 +0000 Subject: [PATCH 11/18] keeping tests relevant to the scope of this pr --- src/py/flwr/common/recordset_test.py | 49 ---------------------------- 1 file changed, 49 deletions(-) diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index 7b381460207f..175454e679b4 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -16,18 +16,15 @@ from contextlib import nullcontext from copy import deepcopy -from functools import partial from typing import Any, Callable, Dict, List, OrderedDict, Type, Union import numpy as np import pytest from .configsrecord import ConfigsRecord -from .flowercontext import FlowerContext, Metadata from .metricsrecord import MetricsRecord from .parameter import ndarrays_to_parameters, parameters_to_ndarrays from .parametersrecord import Array, ParametersRecord -from .recordset import RecordSet from .recordset_utils import ( evaluate_ins_to_recordset, evaluate_res_to_recordset, @@ -561,49 +558,3 @@ def test_get_parameters_res_to_recordset_and_back() -> None: getparameteres_res_ = recordset_to_getparameters_res(recordset) assert getparameters_res_copy == getparameteres_res_ - - -@pytest.mark.parametrize( - "ins, convert_fn, task_type", - [ - (_get_valid_fitins, partial(fit_ins_to_recordset, keep_input=False), "fit_ins"), - ( - _get_valid_evaluateins, - partial(evaluate_ins_to_recordset, keep_input=False), - "evaluate_ins", - ), - ( - _get_valid_getpropertiesins, - getproperties_ins_to_recordset, - "get_properties_ins", - ), - ( - _get_valid_getparametersins, - getparameters_ins_to_recordset, - "get_parameters_ins", - ), - ], -) -def test_flowercontext_driver_to_client( - ins: Union[FitIns, EvaluateIns, GetPropertiesIns, GetParametersIns], - convert_fn: Union[ - Callable[[FitIns], RecordSet], - Callable[[EvaluateIns], RecordSet], - Callable[[GetPropertiesIns], RecordSet], - Callable[[GetParametersIns], RecordSet], - ], - task_type: str, -) -> None: - """.""" - f_context = FlowerContext( - in_message=RecordSet(), - out_message=convert_fn(ins()), - local=RecordSet(), - metadata=Metadata( - run_id=0, task_id="", group_id="", ttl="", task_type=task_type - ), - ) - - # TODO: embedd `f_context` in TaskIns - - # Construct FlowerContext from TaskIns From d9f5b4b86c36e402ec2e30d22eff34c1b1810b91 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Mon, 22 Jan 2024 17:40:33 +0000 Subject: [PATCH 12/18] better --- src/py/flwr/common/recordset_test.py | 72 +++++++++++++++++---------- src/py/flwr/common/recordset_utils.py | 54 +++++--------------- 2 files changed, 58 insertions(+), 68 deletions(-) diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index 175454e679b4..10c044d45a6a 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -387,11 +387,32 @@ def _get_valid_fitins() -> FitIns: return FitIns(parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0}) +def _get_valid_fitres_with_config(metrics: Dict[str, Scalar]) -> FitRes: + """Returnn Valid parameters but potentially invalid config.""" + arrays = get_ndarrays() + return FitRes( + parameters=ndarrays_to_parameters(arrays), + num_examples=1, + status=Status(code=Code(0), message=""), + metrics=metrics, + ) + + def _get_valid_evaluateins() -> EvaluateIns: fit_ins = _get_valid_fitins() return EvaluateIns(parameters=fit_ins.parameters, config=fit_ins.config) +def _get_valid_evaluateres_with_config(metrics: Dict[str, Scalar]) -> EvaluateRes: + """Return potentially invalid config.""" + return EvaluateRes( + num_examples=1, + loss=0.1, + status=Status(code=Code(0), message=""), + metrics=metrics, + ) + + def _get_valid_getparametersins() -> GetParametersIns: config_dict: Dict[str, Scalar] = { "a": 1.0, @@ -402,11 +423,31 @@ def _get_valid_getparametersins() -> GetParametersIns: return GetParametersIns(config_dict) +def _get_valid_getparametersres() -> GetParametersRes: + arrays = get_ndarrays() + return GetParametersRes( + status=Status(code=Code(0), message=""), + parameters=ndarrays_to_parameters(arrays), + ) + + def _get_valid_getpropertiesins() -> GetPropertiesIns: getparamsins = _get_valid_getparametersins() return GetPropertiesIns(config=getparamsins.config) +def _get_valid_getpropertiesres() -> GetPropertiesRes: + config_dict: Dict[str, Scalar] = { + "a": 1.0, + "b": 3, + "c": True, + } # valid since both Ins/Res communicate over ConfigsRecord + + return GetPropertiesRes( + status=Status(code=Code(0), message=""), properties=config_dict + ) + + def test_fitins_to_recordset_and_back() -> None: """Test conversion FitIns --> RecordSet --> FitIns.""" fitins = _get_valid_fitins() @@ -432,13 +473,7 @@ def test_fitins_to_recordset_and_back() -> None: ) def test_fitres_to_recordset_and_back(context: Any, metrics: Dict[str, Scalar]) -> None: """Test conversion FitRes --> RecordSet --> FitRes.""" - arrays = get_ndarrays() - fitres = FitRes( - parameters=ndarrays_to_parameters(arrays), - num_examples=1, - status=Status(code=Code(0), message=""), - metrics=metrics, - ) + fitres = _get_valid_fitres_with_config(metrics) fitres_copy = deepcopy(fitres) @@ -480,12 +515,7 @@ def test_evaluateres_to_recordset_and_back( context: Any, metrics: Dict[str, Scalar] ) -> None: """Test conversion EvaluateRes --> RecordSet --> EvaluateRes.""" - evaluateres = EvaluateRes( - num_examples=1, - loss=0.1, - status=Status(code=Code(0), message=""), - metrics=metrics, - ) + evaluateres = _get_valid_evaluateres_with_config(metrics) evaluateres_copy = deepcopy(evaluateres) @@ -514,15 +544,7 @@ def test_get_properties_ins_to_recordset_and_back() -> None: def test_get_properties_res_to_recordset_and_back() -> None: """Test conversion GetPropertiesRes --> RecordSet --> GetPropertiesRes.""" - config_dict: Dict[str, Scalar] = { - "a": 1.0, - "b": 3, - "c": True, - } # valid since both Ins/Res communicate over ConfigsRecord - - getproperties_res = GetPropertiesRes( - status=Status(code=Code(0), message=""), properties=config_dict - ) + getproperties_res = _get_valid_getpropertiesres() getproperties_res_copy = deepcopy(getproperties_res) @@ -546,11 +568,7 @@ def test_get_parameters_ins_to_recordset_and_back() -> None: def test_get_parameters_res_to_recordset_and_back() -> None: """Test conversion GetParametersRes --> RecordSet --> GetParametersRes.""" - arrays = get_ndarrays() - getparameteres_res = GetParametersRes( - status=Status(code=Code(0), message=""), - parameters=ndarrays_to_parameters(arrays), - ) + getparameteres_res = _get_valid_getparametersres() getparameters_res_copy = deepcopy(getparameteres_res) diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_utils.py index f5a347b677a2..21e3210826ec 100644 --- a/src/py/flwr/common/recordset_utils.py +++ b/src/py/flwr/common/recordset_utils.py @@ -110,32 +110,6 @@ def parameters_to_parametersrecord( return p_record -def _check_mapping_from_scalar_to_metricsrecordstypes( - scalar_config: Dict[str, Scalar], -) -> Dict[str, MetricsRecordValues]: - """.""" - for value in scalar_config.values(): - if not isinstance(value, get_args(MetricsRecordValues)): - raise TypeError( - f"Supported types are {MetricsRecordValues}. " - f"But you used type: {type(value)}" - ) - return cast(Dict[str, MetricsRecordValues], scalar_config) - - -def _check_mapping_from_scalar_to_configsrecordstypes( - scalar_config: Dict[str, Scalar], -) -> Dict[str, ConfigsRecordValues]: - """.""" - for value in scalar_config.values(): - if not isinstance(value, get_args(ConfigsRecordValues)): - raise TypeError( - f"Supported types are {ConfigsRecordValues}. " - f"But you used type: {type(value)}" - ) - return cast(Dict[str, ConfigsRecordValues], scalar_config) - - def _check_mapping_from_recordscalartype_to_scalar( record_data: Mapping[str, Union[ConfigsRecordValues, MetricsRecordValues]] ) -> Dict[str, Scalar]: @@ -184,8 +158,7 @@ def _fit_or_evaluate_ins_to_recordset( record=parameters_to_parametersrecord(ins.parameters, keep_input=keep_input), ) - config = _check_mapping_from_scalar_to_configsrecordstypes(ins.config) - recordset.set_configs(name=f"{ins_str}.config", record=ConfigsRecord(config)) + recordset.set_configs(name=f"{ins_str}.config", record=ConfigsRecord(ins.config)) return recordset @@ -251,8 +224,9 @@ def fit_res_to_recordset(fitres: FitRes, keep_input: bool) -> RecordSet: res_str = "fitres" - metrics = _check_mapping_from_scalar_to_metricsrecordstypes(fitres.metrics) - recordset.set_metrics(name=f"{res_str}.metrics", record=MetricsRecord(metrics)) + recordset.set_metrics( + name=f"{res_str}.metrics", record=MetricsRecord(fitres.metrics) + ) recordset.set_metrics( name=f"{res_str}.num_examples", record=MetricsRecord({"num_examples": fitres.num_examples}), @@ -321,8 +295,9 @@ def evaluate_res_to_recordset(evaluateres: EvaluateRes) -> RecordSet: ) # metrics - metrics = _check_mapping_from_scalar_to_metricsrecordstypes(evaluateres.metrics) - recordset.set_metrics(name=f"{res_str}.metrics", record=MetricsRecord(metrics)) + recordset.set_metrics( + name=f"{res_str}.metrics", record=MetricsRecord(evaluateres.metrics) + ) # status recordset = _embed_status_into_recordset( @@ -345,8 +320,9 @@ def getparameters_ins_to_recordset(getparameters_ins: GetParametersIns) -> Recor """Construct a RecordSet from a GetParametersIns object.""" recordset = RecordSet() - config = _check_mapping_from_scalar_to_configsrecordstypes(getparameters_ins.config) - recordset.set_configs(name="getparametersins.config", record=ConfigsRecord(config)) + recordset.set_configs( + name="getparametersins.config", record=ConfigsRecord(getparameters_ins.config) + ) return recordset @@ -387,11 +363,8 @@ def recordset_to_getproperties_ins(recordset: RecordSet) -> GetPropertiesIns: def getproperties_ins_to_recordset(getpropertiesins: GetPropertiesIns) -> RecordSet: """Construct a RecordSet from a GetPropertiesRes object.""" recordset = RecordSet() - config_dict = _check_mapping_from_scalar_to_configsrecordstypes( - getpropertiesins.config - ) recordset.set_configs( - name="getpropertiesins.config", record=ConfigsRecord(config_dict) + name="getpropertiesins.config", record=ConfigsRecord(getpropertiesins.config) ) return recordset @@ -411,10 +384,9 @@ def getproperties_res_to_recordset(getpropertiesres: GetPropertiesRes) -> Record """Construct a RecordSet from a GetPropertiesRes object.""" recordset = RecordSet() res_str = "getpropertiesres" - configs = _check_mapping_from_scalar_to_configsrecordstypes( - getpropertiesres.properties + recordset.set_configs( + name=f"{res_str}.properties", record=ConfigsRecord(getpropertiesres.properties) ) - recordset.set_configs(name=f"{res_str}.properties", record=ConfigsRecord(configs)) # status recordset = _embed_status_into_recordset( res_str, getpropertiesres.status, recordset From f2c5c5d4c2ab0fe449c4250c07dfa061d9affc24 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Mon, 22 Jan 2024 18:12:04 +0000 Subject: [PATCH 13/18] pass --- src/py/flwr/common/recordset_utils.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_utils.py index 21e3210826ec..daf4409861b2 100644 --- a/src/py/flwr/common/recordset_utils.py +++ b/src/py/flwr/common/recordset_utils.py @@ -158,7 +158,9 @@ def _fit_or_evaluate_ins_to_recordset( record=parameters_to_parametersrecord(ins.parameters, keep_input=keep_input), ) - recordset.set_configs(name=f"{ins_str}.config", record=ConfigsRecord(ins.config)) + recordset.set_configs( + name=f"{ins_str}.config", record=ConfigsRecord(ins.config) # type: ignore + ) return recordset @@ -225,7 +227,7 @@ def fit_res_to_recordset(fitres: FitRes, keep_input: bool) -> RecordSet: res_str = "fitres" recordset.set_metrics( - name=f"{res_str}.metrics", record=MetricsRecord(fitres.metrics) + name=f"{res_str}.metrics", record=MetricsRecord(fitres.metrics) # type: ignore ) recordset.set_metrics( name=f"{res_str}.num_examples", @@ -296,7 +298,8 @@ def evaluate_res_to_recordset(evaluateres: EvaluateRes) -> RecordSet: # metrics recordset.set_metrics( - name=f"{res_str}.metrics", record=MetricsRecord(evaluateres.metrics) + name=f"{res_str}.metrics", + record=MetricsRecord(evaluateres.metrics), # type: ignore ) # status @@ -321,7 +324,8 @@ def getparameters_ins_to_recordset(getparameters_ins: GetParametersIns) -> Recor recordset = RecordSet() recordset.set_configs( - name="getparametersins.config", record=ConfigsRecord(getparameters_ins.config) + name="getparametersins.config", + record=ConfigsRecord(getparameters_ins.config), # type: ignore ) return recordset @@ -364,7 +368,8 @@ def getproperties_ins_to_recordset(getpropertiesins: GetPropertiesIns) -> Record """Construct a RecordSet from a GetPropertiesRes object.""" recordset = RecordSet() recordset.set_configs( - name="getpropertiesins.config", record=ConfigsRecord(getpropertiesins.config) + name="getpropertiesins.config", + record=ConfigsRecord(getpropertiesins.config), # type: ignore ) return recordset @@ -385,7 +390,8 @@ def getproperties_res_to_recordset(getpropertiesres: GetPropertiesRes) -> Record recordset = RecordSet() res_str = "getpropertiesres" recordset.set_configs( - name=f"{res_str}.properties", record=ConfigsRecord(getpropertiesres.properties) + name=f"{res_str}.properties", + record=ConfigsRecord(getpropertiesres.properties), # type: ignore ) # status recordset = _embed_status_into_recordset( From be1ada67cc22f7ad19f09fa654fe7482aa1a9a9e Mon Sep 17 00:00:00 2001 From: jafermarq Date: Tue, 23 Jan 2024 14:46:06 +0000 Subject: [PATCH 14/18] move recordset-legacy tests to a new file --- .../recordset_and_legacy_messages_test.py | 266 ++++++++++++++++++ src/py/flwr/common/recordset_test.py | 233 +-------------- 2 files changed, 267 insertions(+), 232 deletions(-) create mode 100644 src/py/flwr/common/recordset_and_legacy_messages_test.py diff --git a/src/py/flwr/common/recordset_and_legacy_messages_test.py b/src/py/flwr/common/recordset_and_legacy_messages_test.py new file mode 100644 index 000000000000..c6611a9a95d7 --- /dev/null +++ b/src/py/flwr/common/recordset_and_legacy_messages_test.py @@ -0,0 +1,266 @@ +# Copyright 2024 Flower Labs GmbH. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""RecordSet from legacy messages tests.""" + +from contextlib import nullcontext +from copy import deepcopy +from typing import Any, Dict + +import numpy as np +import pytest + +from .parameter import ndarrays_to_parameters +from .recordset_utils import ( + evaluate_ins_to_recordset, + evaluate_res_to_recordset, + fit_ins_to_recordset, + fit_res_to_recordset, + getparameters_ins_to_recordset, + getparameters_res_to_recordset, + getproperties_ins_to_recordset, + getproperties_res_to_recordset, + recordset_to_evaluate_ins, + recordset_to_evaluate_res, + recordset_to_fit_ins, + recordset_to_fit_res, + recordset_to_getparameters_ins, + recordset_to_getparameters_res, + recordset_to_getproperties_ins, + recordset_to_getproperties_res, +) +from .typing import ( + Code, + EvaluateIns, + EvaluateRes, + FitIns, + FitRes, + GetParametersIns, + GetParametersRes, + GetPropertiesIns, + GetPropertiesRes, + NDArrays, + Scalar, + Status, +) + + +def get_ndarrays() -> NDArrays: + """Return list of NumPy arrays.""" + arr1 = np.array([[1.0, 2.0], [3.0, 4], [5.0, 6.0]]) + arr2 = np.eye(2, 7, 3) + + return [arr1, arr2] + + +################################################## +# Testing conversion: *Ins --> RecordSet --> *Ins +# Testing conversion: *Res <-- RecordSet <-- *Res +################################################## + + +def _get_valid_fitins() -> FitIns: + arrays = get_ndarrays() + return FitIns(parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0}) + + +def _get_valid_fitres_with_config(metrics: Dict[str, Scalar]) -> FitRes: + """Returnn Valid parameters but potentially invalid config.""" + arrays = get_ndarrays() + return FitRes( + parameters=ndarrays_to_parameters(arrays), + num_examples=1, + status=Status(code=Code(0), message=""), + metrics=metrics, + ) + + +def _get_valid_evaluateins() -> EvaluateIns: + fit_ins = _get_valid_fitins() + return EvaluateIns(parameters=fit_ins.parameters, config=fit_ins.config) + + +def _get_valid_evaluateres_with_config(metrics: Dict[str, Scalar]) -> EvaluateRes: + """Return potentially invalid config.""" + return EvaluateRes( + num_examples=1, + loss=0.1, + status=Status(code=Code(0), message=""), + metrics=metrics, + ) + + +def _get_valid_getparametersins() -> GetParametersIns: + config_dict: Dict[str, Scalar] = { + "a": 1.0, + "b": 3, + "c": True, + } # valid since both Ins/Res communicate over ConfigsRecord + + return GetParametersIns(config_dict) + + +def _get_valid_getparametersres() -> GetParametersRes: + arrays = get_ndarrays() + return GetParametersRes( + status=Status(code=Code(0), message=""), + parameters=ndarrays_to_parameters(arrays), + ) + + +def _get_valid_getpropertiesins() -> GetPropertiesIns: + getparamsins = _get_valid_getparametersins() + return GetPropertiesIns(config=getparamsins.config) + + +def _get_valid_getpropertiesres() -> GetPropertiesRes: + config_dict: Dict[str, Scalar] = { + "a": 1.0, + "b": 3, + "c": True, + } # valid since both Ins/Res communicate over ConfigsRecord + + return GetPropertiesRes( + status=Status(code=Code(0), message=""), properties=config_dict + ) + + +def test_fitins_to_recordset_and_back() -> None: + """Test conversion FitIns --> RecordSet --> FitIns.""" + fitins = _get_valid_fitins() + + fitins_copy = deepcopy(fitins) + + recordset = fit_ins_to_recordset(fitins, keep_input=False) + + fitins_ = recordset_to_fit_ins(recordset, keep_input=False) + + assert fitins_copy == fitins_ + + +@pytest.mark.parametrize( + "context, metrics", + [ + (nullcontext(), {"a": 1.0, "b": 0}), + ( + pytest.raises(TypeError), + {"a": 1.0, "b": 3, "c": True}, + ), # fails due to unsupported type for metricsrecord value + ], +) +def test_fitres_to_recordset_and_back(context: Any, metrics: Dict[str, Scalar]) -> None: + """Test conversion FitRes --> RecordSet --> FitRes.""" + fitres = _get_valid_fitres_with_config(metrics) + + fitres_copy = deepcopy(fitres) + + with context: + recordset = fit_res_to_recordset(fitres, keep_input=False) + fitres_ = recordset_to_fit_res(recordset, keep_input=False) + + # only check if we didn't test for an invalid setting. Only in valid settings + # makes sense to evaluate the below, since both functions above have succesfully + # being executed. + if isinstance(context, nullcontext): + assert fitres_copy == fitres_ + + +def test_evaluateins_to_recordset_and_back() -> None: + """Test conversion EvaluateIns --> RecordSet --> EvaluateIns.""" + evaluateins = _get_valid_evaluateins() + + evaluateins_copy = deepcopy(evaluateins) + + recordset = evaluate_ins_to_recordset(evaluateins, keep_input=False) + + evaluateins_ = recordset_to_evaluate_ins(recordset, keep_input=False) + + assert evaluateins_copy == evaluateins_ + + +@pytest.mark.parametrize( + "context, metrics", + [ + (nullcontext(), {"a": 1.0, "b": 0}), + ( + pytest.raises(TypeError), + {"a": 1.0, "b": 3, "c": True}, + ), # fails due to unsupported type for metricsrecord value + ], +) +def test_evaluateres_to_recordset_and_back( + context: Any, metrics: Dict[str, Scalar] +) -> None: + """Test conversion EvaluateRes --> RecordSet --> EvaluateRes.""" + evaluateres = _get_valid_evaluateres_with_config(metrics) + + evaluateres_copy = deepcopy(evaluateres) + + with context: + recordset = evaluate_res_to_recordset(evaluateres) + evaluateres_ = recordset_to_evaluate_res(recordset) + + # only check if we didn't test for an invalid setting. Only in valid settings + # makes sense to evaluate the below, since both functions above have succesfully + # being executed. + if isinstance(context, nullcontext): + assert evaluateres_copy == evaluateres_ + + +def test_get_properties_ins_to_recordset_and_back() -> None: + """Test conversion GetPropertiesIns --> RecordSet --> GetPropertiesIns.""" + getproperties_ins = _get_valid_getpropertiesins() + + getproperties_ins_copy = deepcopy(getproperties_ins) + + recordset = getproperties_ins_to_recordset(getproperties_ins) + getproperties_ins_ = recordset_to_getproperties_ins(recordset) + + assert getproperties_ins_copy == getproperties_ins_ + + +def test_get_properties_res_to_recordset_and_back() -> None: + """Test conversion GetPropertiesRes --> RecordSet --> GetPropertiesRes.""" + getproperties_res = _get_valid_getpropertiesres() + + getproperties_res_copy = deepcopy(getproperties_res) + + recordset = getproperties_res_to_recordset(getproperties_res) + getproperties_res_ = recordset_to_getproperties_res(recordset) + + assert getproperties_res_copy == getproperties_res_ + + +def test_get_parameters_ins_to_recordset_and_back() -> None: + """Test conversion GetParametersIns --> RecordSet --> GetParametersIns.""" + getparameters_ins = _get_valid_getparametersins() + + getparameters_ins_copy = deepcopy(getparameters_ins) + + recordset = getparameters_ins_to_recordset(getparameters_ins) + getparameters_ins_ = recordset_to_getparameters_ins(recordset) + + assert getparameters_ins_copy == getparameters_ins_ + + +def test_get_parameters_res_to_recordset_and_back() -> None: + """Test conversion GetParametersRes --> RecordSet --> GetParametersRes.""" + getparameteres_res = _get_valid_getparametersres() + + getparameters_res_copy = deepcopy(getparameteres_res) + + recordset = getparameters_res_to_recordset(getparameteres_res) + getparameteres_res_ = recordset_to_getparameters_res(recordset) + + assert getparameters_res_copy == getparameteres_res_ diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index 10c044d45a6a..f40017ce735c 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -14,9 +14,7 @@ # ============================================================================== """RecordSet tests.""" -from contextlib import nullcontext -from copy import deepcopy -from typing import Any, Callable, Dict, List, OrderedDict, Type, Union +from typing import Callable, Dict, List, OrderedDict, Type, Union import numpy as np import pytest @@ -26,42 +24,15 @@ from .parameter import ndarrays_to_parameters, parameters_to_ndarrays from .parametersrecord import Array, ParametersRecord from .recordset_utils import ( - evaluate_ins_to_recordset, - evaluate_res_to_recordset, - fit_ins_to_recordset, - fit_res_to_recordset, - getparameters_ins_to_recordset, - getparameters_res_to_recordset, - getproperties_ins_to_recordset, - getproperties_res_to_recordset, parameters_to_parametersrecord, parametersrecord_to_parameters, - recordset_to_evaluate_ins, - recordset_to_evaluate_res, - recordset_to_fit_ins, - recordset_to_fit_res, - recordset_to_getparameters_ins, - recordset_to_getparameters_res, - recordset_to_getproperties_ins, - recordset_to_getproperties_res, ) from .typing import ( - Code, ConfigsRecordValues, - EvaluateIns, - EvaluateRes, - FitIns, - FitRes, - GetParametersIns, - GetParametersRes, - GetPropertiesIns, - GetPropertiesRes, MetricsRecordValues, NDArray, NDArrays, Parameters, - Scalar, - Status, ) @@ -374,205 +345,3 @@ def test_set_configs_to_configsrecord_with_incorrect_types( with pytest.raises(TypeError): m_record.set_configs(my_metrics) # type: ignore - - -################################################## -# Testing conversion: *Ins --> RecordSet --> *Ins -# Testing conversion: *Res <-- RecordSet <-- *Res -################################################## - - -def _get_valid_fitins() -> FitIns: - arrays = get_ndarrays() - return FitIns(parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0}) - - -def _get_valid_fitres_with_config(metrics: Dict[str, Scalar]) -> FitRes: - """Returnn Valid parameters but potentially invalid config.""" - arrays = get_ndarrays() - return FitRes( - parameters=ndarrays_to_parameters(arrays), - num_examples=1, - status=Status(code=Code(0), message=""), - metrics=metrics, - ) - - -def _get_valid_evaluateins() -> EvaluateIns: - fit_ins = _get_valid_fitins() - return EvaluateIns(parameters=fit_ins.parameters, config=fit_ins.config) - - -def _get_valid_evaluateres_with_config(metrics: Dict[str, Scalar]) -> EvaluateRes: - """Return potentially invalid config.""" - return EvaluateRes( - num_examples=1, - loss=0.1, - status=Status(code=Code(0), message=""), - metrics=metrics, - ) - - -def _get_valid_getparametersins() -> GetParametersIns: - config_dict: Dict[str, Scalar] = { - "a": 1.0, - "b": 3, - "c": True, - } # valid since both Ins/Res communicate over ConfigsRecord - - return GetParametersIns(config_dict) - - -def _get_valid_getparametersres() -> GetParametersRes: - arrays = get_ndarrays() - return GetParametersRes( - status=Status(code=Code(0), message=""), - parameters=ndarrays_to_parameters(arrays), - ) - - -def _get_valid_getpropertiesins() -> GetPropertiesIns: - getparamsins = _get_valid_getparametersins() - return GetPropertiesIns(config=getparamsins.config) - - -def _get_valid_getpropertiesres() -> GetPropertiesRes: - config_dict: Dict[str, Scalar] = { - "a": 1.0, - "b": 3, - "c": True, - } # valid since both Ins/Res communicate over ConfigsRecord - - return GetPropertiesRes( - status=Status(code=Code(0), message=""), properties=config_dict - ) - - -def test_fitins_to_recordset_and_back() -> None: - """Test conversion FitIns --> RecordSet --> FitIns.""" - fitins = _get_valid_fitins() - - fitins_copy = deepcopy(fitins) - - recordset = fit_ins_to_recordset(fitins, keep_input=False) - - fitins_ = recordset_to_fit_ins(recordset, keep_input=False) - - assert fitins_copy == fitins_ - - -@pytest.mark.parametrize( - "context, metrics", - [ - (nullcontext(), {"a": 1.0, "b": 0}), - ( - pytest.raises(TypeError), - {"a": 1.0, "b": 3, "c": True}, - ), # fails due to unsupported type for metricsrecord value - ], -) -def test_fitres_to_recordset_and_back(context: Any, metrics: Dict[str, Scalar]) -> None: - """Test conversion FitRes --> RecordSet --> FitRes.""" - fitres = _get_valid_fitres_with_config(metrics) - - fitres_copy = deepcopy(fitres) - - with context: - recordset = fit_res_to_recordset(fitres, keep_input=False) - fitres_ = recordset_to_fit_res(recordset, keep_input=False) - - # only check if we didn't test for an invalid setting. Only in valid settings - # makes sense to evaluate the below, since both functions above have succesfully - # being executed. - if isinstance(context, nullcontext): - assert fitres_copy == fitres_ - - -def test_evaluateins_to_recordset_and_back() -> None: - """Test conversion EvaluateIns --> RecordSet --> EvaluateIns.""" - evaluateins = _get_valid_evaluateins() - - evaluateins_copy = deepcopy(evaluateins) - - recordset = evaluate_ins_to_recordset(evaluateins, keep_input=False) - - evaluateins_ = recordset_to_evaluate_ins(recordset, keep_input=False) - - assert evaluateins_copy == evaluateins_ - - -@pytest.mark.parametrize( - "context, metrics", - [ - (nullcontext(), {"a": 1.0, "b": 0}), - ( - pytest.raises(TypeError), - {"a": 1.0, "b": 3, "c": True}, - ), # fails due to unsupported type for metricsrecord value - ], -) -def test_evaluateres_to_recordset_and_back( - context: Any, metrics: Dict[str, Scalar] -) -> None: - """Test conversion EvaluateRes --> RecordSet --> EvaluateRes.""" - evaluateres = _get_valid_evaluateres_with_config(metrics) - - evaluateres_copy = deepcopy(evaluateres) - - with context: - recordset = evaluate_res_to_recordset(evaluateres) - evaluateres_ = recordset_to_evaluate_res(recordset) - - # only check if we didn't test for an invalid setting. Only in valid settings - # makes sense to evaluate the below, since both functions above have succesfully - # being executed. - if isinstance(context, nullcontext): - assert evaluateres_copy == evaluateres_ - - -def test_get_properties_ins_to_recordset_and_back() -> None: - """Test conversion GetPropertiesIns --> RecordSet --> GetPropertiesIns.""" - getproperties_ins = _get_valid_getpropertiesins() - - getproperties_ins_copy = deepcopy(getproperties_ins) - - recordset = getproperties_ins_to_recordset(getproperties_ins) - getproperties_ins_ = recordset_to_getproperties_ins(recordset) - - assert getproperties_ins_copy == getproperties_ins_ - - -def test_get_properties_res_to_recordset_and_back() -> None: - """Test conversion GetPropertiesRes --> RecordSet --> GetPropertiesRes.""" - getproperties_res = _get_valid_getpropertiesres() - - getproperties_res_copy = deepcopy(getproperties_res) - - recordset = getproperties_res_to_recordset(getproperties_res) - getproperties_res_ = recordset_to_getproperties_res(recordset) - - assert getproperties_res_copy == getproperties_res_ - - -def test_get_parameters_ins_to_recordset_and_back() -> None: - """Test conversion GetParametersIns --> RecordSet --> GetParametersIns.""" - getparameters_ins = _get_valid_getparametersins() - - getparameters_ins_copy = deepcopy(getparameters_ins) - - recordset = getparameters_ins_to_recordset(getparameters_ins) - getparameters_ins_ = recordset_to_getparameters_ins(recordset) - - assert getparameters_ins_copy == getparameters_ins_ - - -def test_get_parameters_res_to_recordset_and_back() -> None: - """Test conversion GetParametersRes --> RecordSet --> GetParametersRes.""" - getparameteres_res = _get_valid_getparametersres() - - getparameters_res_copy = deepcopy(getparameteres_res) - - recordset = getparameters_res_to_recordset(getparameteres_res) - getparameteres_res_ = recordset_to_getparameters_res(recordset) - - assert getparameters_res_copy == getparameteres_res_ From fdba77648137f47424fbb3589c97d2a5cb43efcb Mon Sep 17 00:00:00 2001 From: jafermarq Date: Tue, 23 Jan 2024 15:46:53 +0000 Subject: [PATCH 15/18] renamed --- ...ordset_and_legacy_messages_test.py => recordset_utils_test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/py/flwr/common/{recordset_and_legacy_messages_test.py => recordset_utils_test.py} (100%) diff --git a/src/py/flwr/common/recordset_and_legacy_messages_test.py b/src/py/flwr/common/recordset_utils_test.py similarity index 100% rename from src/py/flwr/common/recordset_and_legacy_messages_test.py rename to src/py/flwr/common/recordset_utils_test.py From 6ae022a53cfbabcad9633aa5177febf12295d605 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Tue, 23 Jan 2024 15:49:59 +0000 Subject: [PATCH 16/18] renaming --- src/py/flwr/common/{recordset_utils.py => recordset_compat.py} | 0 .../{recordset_utils_test.py => recordset_compat_test.py} | 2 +- src/py/flwr/common/recordset_test.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/py/flwr/common/{recordset_utils.py => recordset_compat.py} (100%) rename src/py/flwr/common/{recordset_utils_test.py => recordset_compat_test.py} (99%) diff --git a/src/py/flwr/common/recordset_utils.py b/src/py/flwr/common/recordset_compat.py similarity index 100% rename from src/py/flwr/common/recordset_utils.py rename to src/py/flwr/common/recordset_compat.py diff --git a/src/py/flwr/common/recordset_utils_test.py b/src/py/flwr/common/recordset_compat_test.py similarity index 99% rename from src/py/flwr/common/recordset_utils_test.py rename to src/py/flwr/common/recordset_compat_test.py index c6611a9a95d7..914192536f5e 100644 --- a/src/py/flwr/common/recordset_utils_test.py +++ b/src/py/flwr/common/recordset_compat_test.py @@ -22,7 +22,7 @@ import pytest from .parameter import ndarrays_to_parameters -from .recordset_utils import ( +from .recordset_compat import ( evaluate_ins_to_recordset, evaluate_res_to_recordset, fit_ins_to_recordset, diff --git a/src/py/flwr/common/recordset_test.py b/src/py/flwr/common/recordset_test.py index f40017ce735c..e1825eaeef14 100644 --- a/src/py/flwr/common/recordset_test.py +++ b/src/py/flwr/common/recordset_test.py @@ -23,7 +23,7 @@ from .metricsrecord import MetricsRecord from .parameter import ndarrays_to_parameters, parameters_to_ndarrays from .parametersrecord import Array, ParametersRecord -from .recordset_utils import ( +from .recordset_compat import ( parameters_to_parametersrecord, parametersrecord_to_parameters, ) From e884586aee9f9a766df4d757bfc920b5c2a255ff Mon Sep 17 00:00:00 2001 From: jafermarq Date: Tue, 23 Jan 2024 15:59:36 +0000 Subject: [PATCH 17/18] renamings --- src/py/flwr/common/recordset_compat.py | 32 +++++------ src/py/flwr/common/recordset_compat_test.py | 64 ++++++++++----------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/py/flwr/common/recordset_compat.py b/src/py/flwr/common/recordset_compat.py index daf4409861b2..538839c9518c 100644 --- a/src/py/flwr/common/recordset_compat.py +++ b/src/py/flwr/common/recordset_compat.py @@ -184,7 +184,7 @@ def _extract_status_from_recordset(res_str: str, recordset: RecordSet) -> Status return Status(code=Code(code), message=str(status["message"])) -def recordset_to_fit_ins(recordset: RecordSet, keep_input: bool) -> FitIns: +def recordset_to_fitins(recordset: RecordSet, keep_input: bool) -> FitIns: """Derive FitIns from a RecordSet object.""" parameters, config = _recordset_to_fit_or_evaluate_ins_components( recordset, @@ -195,12 +195,12 @@ def recordset_to_fit_ins(recordset: RecordSet, keep_input: bool) -> FitIns: return FitIns(parameters=parameters, config=config) -def fit_ins_to_recordset(fitins: FitIns, keep_input: bool) -> RecordSet: +def fitins_to_recordset(fitins: FitIns, keep_input: bool) -> RecordSet: """Construct a RecordSet from a FitIns object.""" return _fit_or_evaluate_ins_to_recordset(fitins, keep_input) -def recordset_to_fit_res(recordset: RecordSet, keep_input: bool) -> FitRes: +def recordset_to_fitres(recordset: RecordSet, keep_input: bool) -> FitRes: """Derive FitRes from a RecordSet object.""" ins_str = "fitres" parameters = parametersrecord_to_parameters( @@ -220,7 +220,7 @@ def recordset_to_fit_res(recordset: RecordSet, keep_input: bool) -> FitRes: ) -def fit_res_to_recordset(fitres: FitRes, keep_input: bool) -> RecordSet: +def fitres_to_recordset(fitres: FitRes, keep_input: bool) -> RecordSet: """Construct a RecordSet from a FitRes object.""" recordset = RecordSet() @@ -244,7 +244,7 @@ def fit_res_to_recordset(fitres: FitRes, keep_input: bool) -> RecordSet: return recordset -def recordset_to_evaluate_ins(recordset: RecordSet, keep_input: bool) -> EvaluateIns: +def recordset_to_evaluateins(recordset: RecordSet, keep_input: bool) -> EvaluateIns: """Derive EvaluateIns from a RecordSet object.""" parameters, config = _recordset_to_fit_or_evaluate_ins_components( recordset, @@ -255,12 +255,12 @@ def recordset_to_evaluate_ins(recordset: RecordSet, keep_input: bool) -> Evaluat return EvaluateIns(parameters=parameters, config=config) -def evaluate_ins_to_recordset(evaluateins: EvaluateIns, keep_input: bool) -> RecordSet: +def evaluateins_to_recordset(evaluateins: EvaluateIns, keep_input: bool) -> RecordSet: """Construct a RecordSet from a EvaluateIns object.""" return _fit_or_evaluate_ins_to_recordset(evaluateins, keep_input) -def recordset_to_evaluate_res(recordset: RecordSet) -> EvaluateRes: +def recordset_to_evaluateres(recordset: RecordSet) -> EvaluateRes: """Derive EvaluateRes from a RecordSet object.""" ins_str = "evaluateres" @@ -279,7 +279,7 @@ def recordset_to_evaluate_res(recordset: RecordSet) -> EvaluateRes: ) -def evaluate_res_to_recordset(evaluateres: EvaluateRes) -> RecordSet: +def evaluateres_to_recordset(evaluateres: EvaluateRes) -> RecordSet: """Construct a RecordSet from a EvaluateRes object.""" recordset = RecordSet() @@ -310,7 +310,7 @@ def evaluate_res_to_recordset(evaluateres: EvaluateRes) -> RecordSet: return recordset -def recordset_to_getparameters_ins(recordset: RecordSet) -> GetParametersIns: +def recordset_to_getparametersins(recordset: RecordSet) -> GetParametersIns: """Derive GetParametersIns from a RecordSet object.""" config_record = recordset.get_configs("getparametersins.config") @@ -319,7 +319,7 @@ def recordset_to_getparameters_ins(recordset: RecordSet) -> GetParametersIns: return GetParametersIns(config=config_dict) -def getparameters_ins_to_recordset(getparameters_ins: GetParametersIns) -> RecordSet: +def getparametersins_to_recordset(getparameters_ins: GetParametersIns) -> RecordSet: """Construct a RecordSet from a GetParametersIns object.""" recordset = RecordSet() @@ -330,7 +330,7 @@ def getparameters_ins_to_recordset(getparameters_ins: GetParametersIns) -> Recor return recordset -def getparameters_res_to_recordset(getparametersres: GetParametersRes) -> RecordSet: +def getparametersres_to_recordset(getparametersres: GetParametersRes) -> RecordSet: """Construct a RecordSet from a GetParametersRes object.""" recordset = RecordSet() res_str = "getparametersres" @@ -345,7 +345,7 @@ def getparameters_res_to_recordset(getparametersres: GetParametersRes) -> Record return recordset -def recordset_to_getparameters_res(recordset: RecordSet) -> GetParametersRes: +def recordset_to_getparametersres(recordset: RecordSet) -> GetParametersRes: """Derive GetParametersRes from a RecordSet object.""" res_str = "getparametersres" parameters = parametersrecord_to_parameters( @@ -356,7 +356,7 @@ def recordset_to_getparameters_res(recordset: RecordSet) -> GetParametersRes: return GetParametersRes(status=status, parameters=parameters) -def recordset_to_getproperties_ins(recordset: RecordSet) -> GetPropertiesIns: +def recordset_to_getpropertiesins(recordset: RecordSet) -> GetPropertiesIns: """Derive GetPropertiesIns from a RecordSet object.""" config_record = recordset.get_configs("getpropertiesins.config") config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record.data) @@ -364,7 +364,7 @@ def recordset_to_getproperties_ins(recordset: RecordSet) -> GetPropertiesIns: return GetPropertiesIns(config=config_dict) -def getproperties_ins_to_recordset(getpropertiesins: GetPropertiesIns) -> RecordSet: +def getpropertiesins_to_recordset(getpropertiesins: GetPropertiesIns) -> RecordSet: """Construct a RecordSet from a GetPropertiesRes object.""" recordset = RecordSet() recordset.set_configs( @@ -374,7 +374,7 @@ def getproperties_ins_to_recordset(getpropertiesins: GetPropertiesIns) -> Record return recordset -def recordset_to_getproperties_res(recordset: RecordSet) -> GetPropertiesRes: +def recordset_to_getpropertiesres(recordset: RecordSet) -> GetPropertiesRes: """Derive GetPropertiesRes from a RecordSet object.""" res_str = "getpropertiesres" config_record = recordset.get_configs(f"{res_str}.properties") @@ -385,7 +385,7 @@ def recordset_to_getproperties_res(recordset: RecordSet) -> GetPropertiesRes: return GetPropertiesRes(status=status, properties=properties) -def getproperties_res_to_recordset(getpropertiesres: GetPropertiesRes) -> RecordSet: +def getpropertiesres_to_recordset(getpropertiesres: GetPropertiesRes) -> RecordSet: """Construct a RecordSet from a GetPropertiesRes object.""" recordset = RecordSet() res_str = "getpropertiesres" diff --git a/src/py/flwr/common/recordset_compat_test.py b/src/py/flwr/common/recordset_compat_test.py index 914192536f5e..237a5a95fef9 100644 --- a/src/py/flwr/common/recordset_compat_test.py +++ b/src/py/flwr/common/recordset_compat_test.py @@ -23,22 +23,22 @@ from .parameter import ndarrays_to_parameters from .recordset_compat import ( - evaluate_ins_to_recordset, - evaluate_res_to_recordset, - fit_ins_to_recordset, - fit_res_to_recordset, - getparameters_ins_to_recordset, - getparameters_res_to_recordset, - getproperties_ins_to_recordset, - getproperties_res_to_recordset, - recordset_to_evaluate_ins, - recordset_to_evaluate_res, - recordset_to_fit_ins, - recordset_to_fit_res, - recordset_to_getparameters_ins, - recordset_to_getparameters_res, - recordset_to_getproperties_ins, - recordset_to_getproperties_res, + evaluateins_to_recordset, + evaluateres_to_recordset, + fitins_to_recordset, + fitres_to_recordset, + getparametersins_to_recordset, + getparametersres_to_recordset, + getpropertiesins_to_recordset, + getpropertiesres_to_recordset, + recordset_to_evaluateins, + recordset_to_evaluateres, + recordset_to_fitins, + recordset_to_fitres, + recordset_to_getparametersins, + recordset_to_getparametersres, + recordset_to_getpropertiesins, + recordset_to_getpropertiesres, ) from .typing import ( Code, @@ -142,9 +142,9 @@ def test_fitins_to_recordset_and_back() -> None: fitins_copy = deepcopy(fitins) - recordset = fit_ins_to_recordset(fitins, keep_input=False) + recordset = fitins_to_recordset(fitins, keep_input=False) - fitins_ = recordset_to_fit_ins(recordset, keep_input=False) + fitins_ = recordset_to_fitins(recordset, keep_input=False) assert fitins_copy == fitins_ @@ -166,8 +166,8 @@ def test_fitres_to_recordset_and_back(context: Any, metrics: Dict[str, Scalar]) fitres_copy = deepcopy(fitres) with context: - recordset = fit_res_to_recordset(fitres, keep_input=False) - fitres_ = recordset_to_fit_res(recordset, keep_input=False) + recordset = fitres_to_recordset(fitres, keep_input=False) + fitres_ = recordset_to_fitres(recordset, keep_input=False) # only check if we didn't test for an invalid setting. Only in valid settings # makes sense to evaluate the below, since both functions above have succesfully @@ -182,9 +182,9 @@ def test_evaluateins_to_recordset_and_back() -> None: evaluateins_copy = deepcopy(evaluateins) - recordset = evaluate_ins_to_recordset(evaluateins, keep_input=False) + recordset = evaluateins_to_recordset(evaluateins, keep_input=False) - evaluateins_ = recordset_to_evaluate_ins(recordset, keep_input=False) + evaluateins_ = recordset_to_evaluateins(recordset, keep_input=False) assert evaluateins_copy == evaluateins_ @@ -208,8 +208,8 @@ def test_evaluateres_to_recordset_and_back( evaluateres_copy = deepcopy(evaluateres) with context: - recordset = evaluate_res_to_recordset(evaluateres) - evaluateres_ = recordset_to_evaluate_res(recordset) + recordset = evaluateres_to_recordset(evaluateres) + evaluateres_ = recordset_to_evaluateres(recordset) # only check if we didn't test for an invalid setting. Only in valid settings # makes sense to evaluate the below, since both functions above have succesfully @@ -224,8 +224,8 @@ def test_get_properties_ins_to_recordset_and_back() -> None: getproperties_ins_copy = deepcopy(getproperties_ins) - recordset = getproperties_ins_to_recordset(getproperties_ins) - getproperties_ins_ = recordset_to_getproperties_ins(recordset) + recordset = getpropertiesins_to_recordset(getproperties_ins) + getproperties_ins_ = recordset_to_getpropertiesins(recordset) assert getproperties_ins_copy == getproperties_ins_ @@ -236,8 +236,8 @@ def test_get_properties_res_to_recordset_and_back() -> None: getproperties_res_copy = deepcopy(getproperties_res) - recordset = getproperties_res_to_recordset(getproperties_res) - getproperties_res_ = recordset_to_getproperties_res(recordset) + recordset = getpropertiesres_to_recordset(getproperties_res) + getproperties_res_ = recordset_to_getpropertiesres(recordset) assert getproperties_res_copy == getproperties_res_ @@ -248,8 +248,8 @@ def test_get_parameters_ins_to_recordset_and_back() -> None: getparameters_ins_copy = deepcopy(getparameters_ins) - recordset = getparameters_ins_to_recordset(getparameters_ins) - getparameters_ins_ = recordset_to_getparameters_ins(recordset) + recordset = getparametersins_to_recordset(getparameters_ins) + getparameters_ins_ = recordset_to_getparametersins(recordset) assert getparameters_ins_copy == getparameters_ins_ @@ -260,7 +260,7 @@ def test_get_parameters_res_to_recordset_and_back() -> None: getparameters_res_copy = deepcopy(getparameteres_res) - recordset = getparameters_res_to_recordset(getparameteres_res) - getparameteres_res_ = recordset_to_getparameters_res(recordset) + recordset = getparametersres_to_recordset(getparameteres_res) + getparameteres_res_ = recordset_to_getparametersres(recordset) assert getparameters_res_copy == getparameteres_res_ From 21f07a01edd070cc59982894d38aae1aca387807 Mon Sep 17 00:00:00 2001 From: jafermarq Date: Tue, 23 Jan 2024 16:15:39 +0000 Subject: [PATCH 18/18] *res.metrics stored as `ConfigsRecord` --- src/py/flwr/common/recordset_compat.py | 16 +++--- src/py/flwr/common/recordset_compat_test.py | 62 +++++---------------- 2 files changed, 23 insertions(+), 55 deletions(-) diff --git a/src/py/flwr/common/recordset_compat.py b/src/py/flwr/common/recordset_compat.py index 538839c9518c..c45f7fcd9fb8 100644 --- a/src/py/flwr/common/recordset_compat.py +++ b/src/py/flwr/common/recordset_compat.py @@ -210,9 +210,9 @@ def recordset_to_fitres(recordset: RecordSet, keep_input: bool) -> FitRes: num_examples = cast( int, recordset.get_metrics(f"{ins_str}.num_examples")["num_examples"] ) - metrics_record = recordset.get_metrics(f"{ins_str}.metrics") + configs_record = recordset.get_configs(f"{ins_str}.metrics") - metrics = _check_mapping_from_recordscalartype_to_scalar(metrics_record.data) + metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record.data) status = _extract_status_from_recordset(ins_str, recordset) return FitRes( @@ -226,8 +226,8 @@ def fitres_to_recordset(fitres: FitRes, keep_input: bool) -> RecordSet: res_str = "fitres" - recordset.set_metrics( - name=f"{res_str}.metrics", record=MetricsRecord(fitres.metrics) # type: ignore + recordset.set_configs( + name=f"{res_str}.metrics", record=ConfigsRecord(fitres.metrics) # type: ignore ) recordset.set_metrics( name=f"{res_str}.num_examples", @@ -269,9 +269,9 @@ def recordset_to_evaluateres(recordset: RecordSet) -> EvaluateRes: num_examples = cast( int, recordset.get_metrics(f"{ins_str}.num_examples")["num_examples"] ) - metrics_record = recordset.get_metrics(f"{ins_str}.metrics") + configs_record = recordset.get_configs(f"{ins_str}.metrics") - metrics = _check_mapping_from_recordscalartype_to_scalar(metrics_record.data) + metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record.data) status = _extract_status_from_recordset(ins_str, recordset) return EvaluateRes( @@ -297,9 +297,9 @@ def evaluateres_to_recordset(evaluateres: EvaluateRes) -> RecordSet: ) # metrics - recordset.set_metrics( + recordset.set_configs( name=f"{res_str}.metrics", - record=MetricsRecord(evaluateres.metrics), # type: ignore + record=ConfigsRecord(evaluateres.metrics), # type: ignore ) # status diff --git a/src/py/flwr/common/recordset_compat_test.py b/src/py/flwr/common/recordset_compat_test.py index 237a5a95fef9..ad91cd3a42fc 100644 --- a/src/py/flwr/common/recordset_compat_test.py +++ b/src/py/flwr/common/recordset_compat_test.py @@ -14,12 +14,10 @@ # ============================================================================== """RecordSet from legacy messages tests.""" -from contextlib import nullcontext from copy import deepcopy -from typing import Any, Dict +from typing import Dict import numpy as np -import pytest from .parameter import ndarrays_to_parameters from .recordset_compat import ( @@ -75,9 +73,10 @@ def _get_valid_fitins() -> FitIns: return FitIns(parameters=ndarrays_to_parameters(arrays), config={"a": 1.0, "b": 0}) -def _get_valid_fitres_with_config(metrics: Dict[str, Scalar]) -> FitRes: +def _get_valid_fitres() -> FitRes: """Returnn Valid parameters but potentially invalid config.""" arrays = get_ndarrays() + metrics: Dict[str, Scalar] = {"a": 1.0, "b": 0} return FitRes( parameters=ndarrays_to_parameters(arrays), num_examples=1, @@ -91,8 +90,9 @@ def _get_valid_evaluateins() -> EvaluateIns: return EvaluateIns(parameters=fit_ins.parameters, config=fit_ins.config) -def _get_valid_evaluateres_with_config(metrics: Dict[str, Scalar]) -> EvaluateRes: +def _get_valid_evaluateres() -> EvaluateRes: """Return potentially invalid config.""" + metrics: Dict[str, Scalar] = {"a": 1.0, "b": 0} return EvaluateRes( num_examples=1, loss=0.1, @@ -149,31 +149,16 @@ def test_fitins_to_recordset_and_back() -> None: assert fitins_copy == fitins_ -@pytest.mark.parametrize( - "context, metrics", - [ - (nullcontext(), {"a": 1.0, "b": 0}), - ( - pytest.raises(TypeError), - {"a": 1.0, "b": 3, "c": True}, - ), # fails due to unsupported type for metricsrecord value - ], -) -def test_fitres_to_recordset_and_back(context: Any, metrics: Dict[str, Scalar]) -> None: +def test_fitres_to_recordset_and_back() -> None: """Test conversion FitRes --> RecordSet --> FitRes.""" - fitres = _get_valid_fitres_with_config(metrics) + fitres = _get_valid_fitres() fitres_copy = deepcopy(fitres) - with context: - recordset = fitres_to_recordset(fitres, keep_input=False) - fitres_ = recordset_to_fitres(recordset, keep_input=False) + recordset = fitres_to_recordset(fitres, keep_input=False) + fitres_ = recordset_to_fitres(recordset, keep_input=False) - # only check if we didn't test for an invalid setting. Only in valid settings - # makes sense to evaluate the below, since both functions above have succesfully - # being executed. - if isinstance(context, nullcontext): - assert fitres_copy == fitres_ + assert fitres_copy == fitres_ def test_evaluateins_to_recordset_and_back() -> None: @@ -189,33 +174,16 @@ def test_evaluateins_to_recordset_and_back() -> None: assert evaluateins_copy == evaluateins_ -@pytest.mark.parametrize( - "context, metrics", - [ - (nullcontext(), {"a": 1.0, "b": 0}), - ( - pytest.raises(TypeError), - {"a": 1.0, "b": 3, "c": True}, - ), # fails due to unsupported type for metricsrecord value - ], -) -def test_evaluateres_to_recordset_and_back( - context: Any, metrics: Dict[str, Scalar] -) -> None: +def test_evaluateres_to_recordset_and_back() -> None: """Test conversion EvaluateRes --> RecordSet --> EvaluateRes.""" - evaluateres = _get_valid_evaluateres_with_config(metrics) + evaluateres = _get_valid_evaluateres() evaluateres_copy = deepcopy(evaluateres) - with context: - recordset = evaluateres_to_recordset(evaluateres) - evaluateres_ = recordset_to_evaluateres(recordset) + recordset = evaluateres_to_recordset(evaluateres) + evaluateres_ = recordset_to_evaluateres(recordset) - # only check if we didn't test for an invalid setting. Only in valid settings - # makes sense to evaluate the below, since both functions above have succesfully - # being executed. - if isinstance(context, nullcontext): - assert evaluateres_copy == evaluateres_ + assert evaluateres_copy == evaluateres_ def test_get_properties_ins_to_recordset_and_back() -> None: