From 6c39bb9a4c5a0bf2890cc53d6d7db772b9d518f9 Mon Sep 17 00:00:00 2001 From: "Pant, Akshay" Date: Mon, 26 Aug 2024 12:41:54 +0530 Subject: [PATCH 1/8] fix(workspaces): reading shard number from default path Signed-off-by: Pant, Akshay --- openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py | 4 +++- .../keras_cnn_with_compression/src/tfmnist_inmemory.py | 4 +++- openfl-workspace/torch_cnn_histology/src/dataloader.py | 3 ++- .../src/pthistology_inmemory.py | 4 +++- openfl-workspace/torch_cnn_mnist/src/dataloader.py | 4 +++- .../src/ptmnist_inmemory.py | 4 +++- .../torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py | 4 +++- .../torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py | 4 +++- openfl-workspace/torch_unet_kvasir/src/data_loader.py | 6 ++++-- .../torch_unet_kvasir_gramine_ready/src/data_loader.py | 6 ++++-- 10 files changed, 31 insertions(+), 12 deletions(-) diff --git a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py index dec2bc2808..5a3051f53c 100644 --- a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py @@ -3,6 +3,8 @@ """You may copy this file as the starting point of your own model.""" +from re import findall + from openfl.federated import TensorFlowDataLoader from .mnist_utils import load_mnist_shard @@ -28,7 +30,7 @@ def __init__(self, data_path, batch_size, **kwargs): # collaborator list. _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs + shard_num=findall(r'\d+', data_path)[0], **kwargs ) self.X_train = X_train diff --git a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py index dec2bc2808..5a3051f53c 100644 --- a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py @@ -3,6 +3,8 @@ """You may copy this file as the starting point of your own model.""" +from re import findall + from openfl.federated import TensorFlowDataLoader from .mnist_utils import load_mnist_shard @@ -28,7 +30,7 @@ def __init__(self, data_path, batch_size, **kwargs): # collaborator list. _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs + shard_num=findall(r'\d+', data_path)[0], **kwargs ) self.X_train = X_train diff --git a/openfl-workspace/torch_cnn_histology/src/dataloader.py b/openfl-workspace/torch_cnn_histology/src/dataloader.py index dc7dd4e0c6..8cf1652bb0 100644 --- a/openfl-workspace/torch_cnn_histology/src/dataloader.py +++ b/openfl-workspace/torch_cnn_histology/src/dataloader.py @@ -9,6 +9,7 @@ from pathlib import Path from urllib.request import urlretrieve from zipfile import ZipFile +from re import findall from openfl.federated import PyTorchDataLoader import numpy as np @@ -38,7 +39,7 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, random_seed=0, **kwargs) _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( - shard_num=int(data_path), **kwargs) + shard_num=findall(r'\d+', data_path)[0], **kwargs) self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py index 3ebecc2123..9da47eec7c 100644 --- a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py +++ b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py @@ -3,6 +3,8 @@ """You may copy this file as the starting point of your own model.""" +from re import findall + from openfl.federated import PyTorchDataLoader from .histology_utils import load_histology_shard @@ -22,7 +24,7 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, random_seed=0, **kwargs) _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( - shard_num=int(data_path), **kwargs) + shard_num=findall(r'\d+', data_path)[0], **kwargs) self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_mnist/src/dataloader.py b/openfl-workspace/torch_cnn_mnist/src/dataloader.py index 4130ceafdb..d18bf146d6 100644 --- a/openfl-workspace/torch_cnn_mnist/src/dataloader.py +++ b/openfl-workspace/torch_cnn_mnist/src/dataloader.py @@ -3,6 +3,8 @@ """You may copy this file as the starting point of your own model.""" +from re import findall + from openfl.federated import PyTorchDataLoader from torchvision import datasets from torchvision import transforms @@ -27,7 +29,7 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, **kwargs) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs + shard_num=findall(r'\d+', data_path)[0], **kwargs ) self.X_train = X_train diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py index 6ff9d6c5d1..1770de4c21 100644 --- a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py @@ -3,6 +3,8 @@ """You may copy this file as the starting point of your own model.""" +from re import findall + from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard @@ -28,7 +30,7 @@ def __init__(self, data_path, batch_size, **kwargs): # of collaborator list. num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs) + shard_num=findall(r'\d+', data_path)[0], **kwargs) self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py index b570ef8bfb..19fcaecc01 100644 --- a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py @@ -3,6 +3,8 @@ """You may copy this file as the starting point of your own model.""" +from re import findall + from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard @@ -28,7 +30,7 @@ def __init__(self, data_path, batch_size, **kwargs): # of collaborator list. num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs) + shard_num=findall(r'\d+', data_path)[0], **kwargs) self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py index 6ff9d6c5d1..1770de4c21 100644 --- a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py @@ -3,6 +3,8 @@ """You may copy this file as the starting point of your own model.""" +from re import findall + from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard @@ -28,7 +30,7 @@ def __init__(self, data_path, batch_size, **kwargs): # of collaborator list. num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs) + shard_num=findall(r'\d+', data_path)[0], **kwargs) self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_unet_kvasir/src/data_loader.py b/openfl-workspace/torch_unet_kvasir/src/data_loader.py index 15db0d63c1..31b5301c6c 100644 --- a/openfl-workspace/torch_unet_kvasir/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir/src/data_loader.py @@ -6,6 +6,8 @@ import zipfile from os import listdir from pathlib import Path +from re import findall + import numpy as np import PIL @@ -122,8 +124,8 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, **kwargs) load_kvasir_dataset() - self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) - self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) + self.valid_dataset = KvasirDataset(True, shard_num=findall(r'\d+', data_path)[0], **kwargs) + self.train_dataset = KvasirDataset(False, shard_num=findall(r'\d+', data_path)[0], **kwargs) self.train_loader = self.get_train_loader() self.val_loader = self.get_valid_loader() self.batch_size = batch_size diff --git a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py index 15db0d63c1..31b5301c6c 100644 --- a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py @@ -6,6 +6,8 @@ import zipfile from os import listdir from pathlib import Path +from re import findall + import numpy as np import PIL @@ -122,8 +124,8 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, **kwargs) load_kvasir_dataset() - self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) - self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) + self.valid_dataset = KvasirDataset(True, shard_num=findall(r'\d+', data_path)[0], **kwargs) + self.train_dataset = KvasirDataset(False, shard_num=findall(r'\d+', data_path)[0], **kwargs) self.train_loader = self.get_train_loader() self.val_loader = self.get_valid_loader() self.batch_size = batch_size From ab9884ef23547c8691e19a417197cd79f0080632 Mon Sep 17 00:00:00 2001 From: "Pant, Akshay" Date: Mon, 26 Aug 2024 14:41:00 +0530 Subject: [PATCH 2/8] fix(workspaces): added valueerror handling where data path represents shard num Signed-off-by: Pant, Akshay --- .../keras_cnn_mnist/src/tfmnist_inmemory.py | 15 ++++++++++----- .../src/tfmnist_inmemory.py | 15 ++++++++++----- .../torch_cnn_histology/src/dataloader.py | 10 +++++++--- .../src/pthistology_inmemory.py | 13 +++++++++---- .../torch_cnn_mnist/src/dataloader.py | 13 ++++++++----- .../src/ptmnist_inmemory.py | 14 ++++++++++---- .../src/ptmnist_inmemory.py | 14 ++++++++++---- .../src/ptmnist_inmemory.py | 14 ++++++++++---- .../torch_unet_kvasir/src/data_loader.py | 13 ++++++++++--- .../src/data_loader.py | 13 ++++++++++--- 10 files changed, 94 insertions(+), 40 deletions(-) diff --git a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py index 5a3051f53c..e8a05d230a 100644 --- a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py @@ -3,10 +3,11 @@ """You may copy this file as the starting point of your own model.""" -from re import findall - from openfl.federated import TensorFlowDataLoader from .mnist_utils import load_mnist_shard +from logging import getLogger + +logger = getLogger(__name__) class TensorFlowMNISTInMemory(TensorFlowDataLoader): @@ -29,9 +30,13 @@ def __init__(self, data_path, batch_size, **kwargs): # Then we have a way to automatically shard based on rank and size of # collaborator list. - _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=findall(r'\d+', data_path)[0], **kwargs - ) + try: + _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py index 5a3051f53c..e8a05d230a 100644 --- a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py @@ -3,10 +3,11 @@ """You may copy this file as the starting point of your own model.""" -from re import findall - from openfl.federated import TensorFlowDataLoader from .mnist_utils import load_mnist_shard +from logging import getLogger + +logger = getLogger(__name__) class TensorFlowMNISTInMemory(TensorFlowDataLoader): @@ -29,9 +30,13 @@ def __init__(self, data_path, batch_size, **kwargs): # Then we have a way to automatically shard based on rank and size of # collaborator list. - _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=findall(r'\d+', data_path)[0], **kwargs - ) + try: + _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_histology/src/dataloader.py b/openfl-workspace/torch_cnn_histology/src/dataloader.py index 8cf1652bb0..65ec8987c6 100644 --- a/openfl-workspace/torch_cnn_histology/src/dataloader.py +++ b/openfl-workspace/torch_cnn_histology/src/dataloader.py @@ -9,7 +9,6 @@ from pathlib import Path from urllib.request import urlretrieve from zipfile import ZipFile -from re import findall from openfl.federated import PyTorchDataLoader import numpy as np @@ -38,8 +37,13 @@ def __init__(self, data_path, batch_size, **kwargs): """ super().__init__(batch_size, random_seed=0, **kwargs) - _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( - shard_num=findall(r'\d+', data_path)[0], **kwargs) + try: + _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( + shard_num=int(data_path), **kwargs + ) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py index 9da47eec7c..b0eb72557f 100644 --- a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py +++ b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py @@ -3,10 +3,11 @@ """You may copy this file as the starting point of your own model.""" -from re import findall - from openfl.federated import PyTorchDataLoader from .histology_utils import load_histology_shard +from logging import getLogger + +logger = getLogger(__name__) class PyTorchHistologyInMemory(PyTorchDataLoader): @@ -23,8 +24,12 @@ def __init__(self, data_path, batch_size, **kwargs): """ super().__init__(batch_size, random_seed=0, **kwargs) - _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( - shard_num=findall(r'\d+', data_path)[0], **kwargs) + try: + _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( + shard_num=int(data_path), **kwargs) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_mnist/src/dataloader.py b/openfl-workspace/torch_cnn_mnist/src/dataloader.py index d18bf146d6..c3656698b6 100644 --- a/openfl-workspace/torch_cnn_mnist/src/dataloader.py +++ b/openfl-workspace/torch_cnn_mnist/src/dataloader.py @@ -3,8 +3,6 @@ """You may copy this file as the starting point of your own model.""" -from re import findall - from openfl.federated import PyTorchDataLoader from torchvision import datasets from torchvision import transforms @@ -28,9 +26,14 @@ def __init__(self, data_path, batch_size, **kwargs): """ super().__init__(batch_size, **kwargs) - num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=findall(r'\d+', data_path)[0], **kwargs - ) + try: + num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return + self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py index 1770de4c21..32d50b9b8d 100644 --- a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py @@ -3,10 +3,11 @@ """You may copy this file as the starting point of your own model.""" -from re import findall - from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard +from logging import getLogger + +logger = getLogger(__name__) class PyTorchMNISTInMemory(PyTorchDataLoader): @@ -29,8 +30,13 @@ def __init__(self, data_path, batch_size, **kwargs): # Then we have a way to automatically shard based on rank and size # of collaborator list. - num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=findall(r'\d+', data_path)[0], **kwargs) + try: + num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py index 19fcaecc01..863a5c2793 100644 --- a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py @@ -3,10 +3,11 @@ """You may copy this file as the starting point of your own model.""" -from re import findall - from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard +from logging import getLogger + +logger = getLogger(__name__) class PyTorchMNISTInMemory(PyTorchDataLoader): @@ -29,8 +30,13 @@ def __init__(self, data_path, batch_size, **kwargs): # Then we have a way to automatically shard based on rank and size # of collaborator list. - num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=findall(r'\d+', data_path)[0], **kwargs) + try: + num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py index 1770de4c21..32d50b9b8d 100644 --- a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py @@ -3,10 +3,11 @@ """You may copy this file as the starting point of your own model.""" -from re import findall - from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard +from logging import getLogger + +logger = getLogger(__name__) class PyTorchMNISTInMemory(PyTorchDataLoader): @@ -29,8 +30,13 @@ def __init__(self, data_path, batch_size, **kwargs): # Then we have a way to automatically shard based on rank and size # of collaborator list. - num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=findall(r'\d+', data_path)[0], **kwargs) + try: + num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_unet_kvasir/src/data_loader.py b/openfl-workspace/torch_unet_kvasir/src/data_loader.py index 31b5301c6c..b34fac0f99 100644 --- a/openfl-workspace/torch_unet_kvasir/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir/src/data_loader.py @@ -6,7 +6,6 @@ import zipfile from os import listdir from pathlib import Path -from re import findall import numpy as np @@ -20,6 +19,9 @@ from openfl.federated import PyTorchDataLoader from openfl.utilities import validate_file_hash +from logging import getLogger + +logger = getLogger(__name__) def read_data(image_path, mask_path): @@ -124,8 +126,13 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, **kwargs) load_kvasir_dataset() - self.valid_dataset = KvasirDataset(True, shard_num=findall(r'\d+', data_path)[0], **kwargs) - self.train_dataset = KvasirDataset(False, shard_num=findall(r'\d+', data_path)[0], **kwargs) + try: + self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) + self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return + self.train_loader = self.get_train_loader() self.val_loader = self.get_valid_loader() self.batch_size = batch_size diff --git a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py index 31b5301c6c..b34fac0f99 100644 --- a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py @@ -6,7 +6,6 @@ import zipfile from os import listdir from pathlib import Path -from re import findall import numpy as np @@ -20,6 +19,9 @@ from openfl.federated import PyTorchDataLoader from openfl.utilities import validate_file_hash +from logging import getLogger + +logger = getLogger(__name__) def read_data(image_path, mask_path): @@ -124,8 +126,13 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, **kwargs) load_kvasir_dataset() - self.valid_dataset = KvasirDataset(True, shard_num=findall(r'\d+', data_path)[0], **kwargs) - self.train_dataset = KvasirDataset(False, shard_num=findall(r'\d+', data_path)[0], **kwargs) + try: + self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) + self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) + except ValueError: + logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") + return + self.train_loader = self.get_train_loader() self.val_loader = self.get_valid_loader() self.batch_size = batch_size From aded759b1b876463a4bf764ee95b56d23141bc6d Mon Sep 17 00:00:00 2001 From: "Pant, Akshay" Date: Mon, 26 Aug 2024 14:42:20 +0530 Subject: [PATCH 3/8] fix(docs): removed optional from collaborator create command as omitting it leads to an error when starting the collaborator for some templates Signed-off-by: Pant, Akshay --- docs/about/features_index/taskrunner.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst index 034163de02..ca9583e018 100644 --- a/docs/about/features_index/taskrunner.rst +++ b/docs/about/features_index/taskrunner.rst @@ -382,7 +382,7 @@ Importing the Workspace .. code-block:: console - $ fx collaborator create -n {COL_LABEL} -d {DATA_PATH:optional} + $ fx collaborator create -n {COL_LABEL} -d {DATA_PATH} $ fx collaborator generate-cert-request -n {COL_LABEL} From 93e8d9cc829ee84631124583ac813f1066ffe32b Mon Sep 17 00:00:00 2001 From: "Pant, Akshay" Date: Mon, 26 Aug 2024 15:28:17 +0530 Subject: [PATCH 4/8] fix(openfl-workspaces): added type check for data path as shard num (int) before loading dataset Signed-off-by: Pant, Akshay --- .../keras_cnn_mnist/src/tfmnist_inmemory.py | 17 +++++++---------- .../src/tfmnist_inmemory.py | 16 ++++++---------- .../torch_cnn_histology/src/dataloader.py | 12 ++++++------ .../src/pthistology_inmemory.py | 15 +++++++-------- .../torch_cnn_mnist/src/dataloader.py | 13 ++++++------- .../src/ptmnist_inmemory.py | 15 ++++++--------- .../src/ptmnist_inmemory.py | 15 ++++++--------- .../src/ptmnist_inmemory.py | 16 ++++++---------- .../torch_unet_kvasir/src/data_loader.py | 17 +++++++---------- .../src/data_loader.py | 16 +++++++--------- 10 files changed, 64 insertions(+), 88 deletions(-) diff --git a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py index e8a05d230a..33b4cc5ea9 100644 --- a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py @@ -5,9 +5,6 @@ from openfl.federated import TensorFlowDataLoader from .mnist_utils import load_mnist_shard -from logging import getLogger - -logger = getLogger(__name__) class TensorFlowMNISTInMemory(TensorFlowDataLoader): @@ -29,14 +26,14 @@ def __init__(self, data_path, batch_size, **kwargs): # what index/rank is this collaborator. # Then we have a way to automatically shard based on rank and size of # collaborator list. - try: - _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs - ) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + + _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py index e8a05d230a..0825139145 100644 --- a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py @@ -5,9 +5,6 @@ from openfl.federated import TensorFlowDataLoader from .mnist_utils import load_mnist_shard -from logging import getLogger - -logger = getLogger(__name__) class TensorFlowMNISTInMemory(TensorFlowDataLoader): @@ -29,15 +26,14 @@ def __init__(self, data_path, batch_size, **kwargs): # what index/rank is this collaborator. # Then we have a way to automatically shard based on rank and size of # collaborator list. - try: - _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs - ) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) self.X_train = X_train self.y_train = y_train self.X_valid = X_valid diff --git a/openfl-workspace/torch_cnn_histology/src/dataloader.py b/openfl-workspace/torch_cnn_histology/src/dataloader.py index 65ec8987c6..4d5b7faf2b 100644 --- a/openfl-workspace/torch_cnn_histology/src/dataloader.py +++ b/openfl-workspace/torch_cnn_histology/src/dataloader.py @@ -38,13 +38,13 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, random_seed=0, **kwargs) try: - _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( - shard_num=int(data_path), **kwargs - ) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( + shard_num=int(data_path), **kwargs + ) self.X_train = X_train self.y_train = y_train self.X_valid = X_valid diff --git a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py index b0eb72557f..b65ad143d9 100644 --- a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py +++ b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py @@ -5,9 +5,6 @@ from openfl.federated import PyTorchDataLoader from .histology_utils import load_histology_shard -from logging import getLogger - -logger = getLogger(__name__) class PyTorchHistologyInMemory(PyTorchDataLoader): @@ -25,11 +22,13 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, random_seed=0, **kwargs) try: - _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( - shard_num=int(data_path), **kwargs) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + + _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( + shard_num=int(data_path), **kwargs + ) self.X_train = X_train self.y_train = y_train diff --git a/openfl-workspace/torch_cnn_mnist/src/dataloader.py b/openfl-workspace/torch_cnn_mnist/src/dataloader.py index c3656698b6..6d8f490f87 100644 --- a/openfl-workspace/torch_cnn_mnist/src/dataloader.py +++ b/openfl-workspace/torch_cnn_mnist/src/dataloader.py @@ -27,14 +27,13 @@ def __init__(self, data_path, batch_size, **kwargs): super().__init__(batch_size, **kwargs) try: - num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs - ) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return - + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) self.X_train = X_train self.y_train = y_train self.train_loader = self.get_train_loader() diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py index 32d50b9b8d..47842926be 100644 --- a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py @@ -5,9 +5,6 @@ from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard -from logging import getLogger - -logger = getLogger(__name__) class PyTorchMNISTInMemory(PyTorchDataLoader): @@ -31,13 +28,13 @@ def __init__(self, data_path, batch_size, **kwargs): # of collaborator list. try: - num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs - ) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) self.X_train = X_train self.y_train = y_train self.train_loader = self.get_train_loader() diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py index 863a5c2793..33b3e2cbb8 100644 --- a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py @@ -5,9 +5,6 @@ from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard -from logging import getLogger - -logger = getLogger(__name__) class PyTorchMNISTInMemory(PyTorchDataLoader): @@ -31,13 +28,13 @@ def __init__(self, data_path, batch_size, **kwargs): # of collaborator list. try: - num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs - ) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) self.X_train = X_train self.y_train = y_train self.train_loader = self.get_train_loader() diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py index 32d50b9b8d..b6df61fe3b 100644 --- a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py @@ -5,9 +5,6 @@ from openfl.federated import PyTorchDataLoader from .mnist_utils import load_mnist_shard -from logging import getLogger - -logger = getLogger(__name__) class PyTorchMNISTInMemory(PyTorchDataLoader): @@ -29,15 +26,14 @@ def __init__(self, data_path, batch_size, **kwargs): # what index/rank is this collaborator. # Then we have a way to automatically shard based on rank and size # of collaborator list. - try: - num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs - ) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) self.X_train = X_train self.y_train = y_train self.train_loader = self.get_train_loader() diff --git a/openfl-workspace/torch_unet_kvasir/src/data_loader.py b/openfl-workspace/torch_unet_kvasir/src/data_loader.py index b34fac0f99..32012fd108 100644 --- a/openfl-workspace/torch_unet_kvasir/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir/src/data_loader.py @@ -19,9 +19,6 @@ from openfl.federated import PyTorchDataLoader from openfl.utilities import validate_file_hash -from logging import getLogger - -logger = getLogger(__name__) def read_data(image_path, mask_path): @@ -125,14 +122,14 @@ def __init__(self, data_path, batch_size, **kwargs): """ super().__init__(batch_size, **kwargs) - load_kvasir_dataset() try: - self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) - self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return - + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + + load_kvasir_dataset() + self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) + self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) self.train_loader = self.get_train_loader() self.val_loader = self.get_valid_loader() self.batch_size = batch_size diff --git a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py index b34fac0f99..2a646ddb5e 100644 --- a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py @@ -19,9 +19,6 @@ from openfl.federated import PyTorchDataLoader from openfl.utilities import validate_file_hash -from logging import getLogger - -logger = getLogger(__name__) def read_data(image_path, mask_path): @@ -125,13 +122,14 @@ def __init__(self, data_path, batch_size, **kwargs): """ super().__init__(batch_size, **kwargs) - load_kvasir_dataset() try: - self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) - self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) - except ValueError: - logger.error("Please pass the shard number (integer) for the collaborator using data path flag.") - return + int(data_path) + except: + raise ValueError("Pass shard number using data path flag as an int.") + + load_kvasir_dataset() + self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) + self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) self.train_loader = self.get_train_loader() self.val_loader = self.get_valid_loader() From c6e35bf304c47b1ad02e73ad4f0a294bbb53721c Mon Sep 17 00:00:00 2001 From: "Pant, Akshay" Date: Tue, 27 Aug 2024 10:30:13 +0530 Subject: [PATCH 5/8] fix(openfl-workspaces): changed error message for ValueError Signed-off-by: Pant, Akshay --- openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py | 2 +- .../keras_cnn_with_compression/src/tfmnist_inmemory.py | 2 +- openfl-workspace/torch_cnn_histology/src/dataloader.py | 2 +- .../src/pthistology_inmemory.py | 2 +- openfl-workspace/torch_cnn_mnist/src/dataloader.py | 2 +- .../torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py | 2 +- .../torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py | 2 +- .../torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py | 2 +- openfl-workspace/torch_unet_kvasir/src/data_loader.py | 2 +- .../torch_unet_kvasir_gramine_ready/src/data_loader.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py index 33b4cc5ea9..35d20a9e32 100644 --- a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py @@ -29,7 +29,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py index 0825139145..ceadcbe56e 100644 --- a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py @@ -29,7 +29,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_histology/src/dataloader.py b/openfl-workspace/torch_cnn_histology/src/dataloader.py index 4d5b7faf2b..64b425963c 100644 --- a/openfl-workspace/torch_cnn_histology/src/dataloader.py +++ b/openfl-workspace/torch_cnn_histology/src/dataloader.py @@ -40,7 +40,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py index b65ad143d9..5410b8a608 100644 --- a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py +++ b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py @@ -24,7 +24,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_mnist/src/dataloader.py b/openfl-workspace/torch_cnn_mnist/src/dataloader.py index 6d8f490f87..51e19cd6c7 100644 --- a/openfl-workspace/torch_cnn_mnist/src/dataloader.py +++ b/openfl-workspace/torch_cnn_mnist/src/dataloader.py @@ -29,7 +29,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py index 47842926be..0e8e8dab3a 100644 --- a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py @@ -30,7 +30,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py index 33b3e2cbb8..b5ecc91509 100644 --- a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py @@ -30,7 +30,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py index b6df61fe3b..e9021bdcdc 100644 --- a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py @@ -29,7 +29,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_unet_kvasir/src/data_loader.py b/openfl-workspace/torch_unet_kvasir/src/data_loader.py index 32012fd108..f854ebdbc0 100644 --- a/openfl-workspace/torch_unet_kvasir/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir/src/data_loader.py @@ -125,7 +125,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) load_kvasir_dataset() self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) diff --git a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py index 2a646ddb5e..695c9ca789 100644 --- a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py @@ -125,7 +125,7 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Pass shard number using data path flag as an int.") + raise ValueError("Expected `%s` to be representable as `int`.", data_path) load_kvasir_dataset() self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) From 70623916d80499b1e6ec89b465570942c9de42ad Mon Sep 17 00:00:00 2001 From: "Pant, Akshay" Date: Thu, 17 Oct 2024 11:51:48 +0530 Subject: [PATCH 6/8] restructure: linter changes Signed-off-by: Pant, Akshay --- openfl-workspace/torch_unet_kvasir/src/data_loader.py | 2 +- .../torch_unet_kvasir_gramine_ready/src/data_loader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openfl-workspace/torch_unet_kvasir/src/data_loader.py b/openfl-workspace/torch_unet_kvasir/src/data_loader.py index f854ebdbc0..c60da9d559 100644 --- a/openfl-workspace/torch_unet_kvasir/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir/src/data_loader.py @@ -126,7 +126,7 @@ def __init__(self, data_path, batch_size, **kwargs): int(data_path) except: raise ValueError("Expected `%s` to be representable as `int`.", data_path) - + load_kvasir_dataset() self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) diff --git a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py index 695c9ca789..39947e3b9e 100644 --- a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py @@ -126,7 +126,7 @@ def __init__(self, data_path, batch_size, **kwargs): int(data_path) except: raise ValueError("Expected `%s` to be representable as `int`.", data_path) - + load_kvasir_dataset() self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) self.train_dataset = KvasirDataset(False, shard_num=int(data_path), **kwargs) From ca90679af5ca4788e1bc8886dda550dd34f57a0d Mon Sep 17 00:00:00 2001 From: "Pant, Akshay" Date: Mon, 4 Nov 2024 12:51:31 +0530 Subject: [PATCH 7/8] fix(task runner examples): add a more descriptive error message Signed-off-by: Pant, Akshay --- openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py | 6 +++++- .../keras_cnn_with_compression/src/tfmnist_inmemory.py | 6 +++++- openfl-workspace/torch_cnn_histology/src/dataloader.py | 6 +++++- .../src/pthistology_inmemory.py | 6 +++++- openfl-workspace/torch_cnn_mnist/src/dataloader.py | 6 +++++- .../src/ptmnist_inmemory.py | 6 +++++- .../torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py | 6 +++++- .../torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py | 6 +++++- openfl-workspace/torch_unet_kvasir/src/data_loader.py | 6 +++++- .../torch_unet_kvasir_gramine_ready/src/data_loader.py | 6 +++++- 10 files changed, 50 insertions(+), 10 deletions(-) diff --git a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py index 35d20a9e32..51f4ccf739 100644 --- a/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_mnist/src/tfmnist_inmemory.py @@ -29,7 +29,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py index ceadcbe56e..80b913e5f5 100644 --- a/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py +++ b/openfl-workspace/keras_cnn_with_compression/src/tfmnist_inmemory.py @@ -29,7 +29,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_histology/src/dataloader.py b/openfl-workspace/torch_cnn_histology/src/dataloader.py index 64b425963c..fa4ae86778 100644 --- a/openfl-workspace/torch_cnn_histology/src/dataloader.py +++ b/openfl-workspace/torch_cnn_histology/src/dataloader.py @@ -40,7 +40,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py index 5410b8a608..00bc04c7c1 100644 --- a/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py +++ b/openfl-workspace/torch_cnn_histology_gramine_ready/src/pthistology_inmemory.py @@ -24,7 +24,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) _, num_classes, X_train, y_train, X_valid, y_valid = load_histology_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_mnist/src/dataloader.py b/openfl-workspace/torch_cnn_mnist/src/dataloader.py index 51e19cd6c7..3f3eeeb0bb 100644 --- a/openfl-workspace/torch_cnn_mnist/src/dataloader.py +++ b/openfl-workspace/torch_cnn_mnist/src/dataloader.py @@ -29,7 +29,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py index 0e8e8dab3a..74c8ec5d03 100644 --- a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py @@ -30,7 +30,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py index b5ecc91509..324545a763 100644 --- a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py @@ -30,7 +30,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py index e9021bdcdc..ccf234239c 100644 --- a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py +++ b/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py @@ -29,7 +29,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( shard_num=int(data_path), **kwargs diff --git a/openfl-workspace/torch_unet_kvasir/src/data_loader.py b/openfl-workspace/torch_unet_kvasir/src/data_loader.py index c60da9d559..0f968808e4 100644 --- a/openfl-workspace/torch_unet_kvasir/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir/src/data_loader.py @@ -125,7 +125,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) load_kvasir_dataset() self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) diff --git a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py index 39947e3b9e..82bbedeb80 100644 --- a/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py +++ b/openfl-workspace/torch_unet_kvasir_gramine_ready/src/data_loader.py @@ -125,7 +125,11 @@ def __init__(self, data_path, batch_size, **kwargs): try: int(data_path) except: - raise ValueError("Expected `%s` to be representable as `int`.", data_path) + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) load_kvasir_dataset() self.valid_dataset = KvasirDataset(True, shard_num=int(data_path), **kwargs) From aae8185f8ab1ba1ce58cd4ff72bc9931b926bb73 Mon Sep 17 00:00:00 2001 From: "Pant, Akshay" Date: Mon, 4 Nov 2024 12:53:05 +0530 Subject: [PATCH 8/8] fix(task runner docs): revert to DATA_PATH being optional Signed-off-by: Pant, Akshay --- docs/about/features_index/taskrunner.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst index 7b50f36c45..4e67622088 100644 --- a/docs/about/features_index/taskrunner.rst +++ b/docs/about/features_index/taskrunner.rst @@ -382,7 +382,7 @@ Importing the Workspace .. code-block:: console - $ fx collaborator create -n {COL_LABEL} -d {DATA_PATH} + $ fx collaborator create -n {COL_LABEL} -d {DATA_PATH:optional} $ fx collaborator generate-cert-request -n {COL_LABEL}