From 5ab80718bbf3892d2a37413b1c5579531c8ae0fc Mon Sep 17 00:00:00 2001 From: JMGaljaard Date: Fri, 2 Sep 2022 12:49:09 +0200 Subject: [PATCH] Update configuration object examples for new definition --- .../example_arrival_config.json | 125 ++-- configs/example_cloud_experiment.json | 7 +- .../{distributed => federated}/__init__.py | 0 .../{distributed => federated}/cifar10.py | 2 +- .../{distributed => federated}/cifar100.py | 0 .../{distributed => federated}/dataset.py | 2 - .../fashion_mnist.py | 0 .../{distributed => federated}/mnist.py | 0 fltk/datasets/loader_util.py | 47 -- fltk/util/generate_data_distribution.py | 70 --- fltk/util/generate_docker_compose.py | 574 ------------------ fltk/util/generate_docker_compose_2.py | 136 ----- fltk/util/generate_experiments.py | 168 ----- fltk/util/offloading_estimate.py | 61 -- jupyter/terraform-kubeflow/main.tf | 3 +- 15 files changed, 77 insertions(+), 1118 deletions(-) rename fltk/datasets/{distributed => federated}/__init__.py (100%) rename fltk/datasets/{distributed => federated}/cifar10.py (97%) rename fltk/datasets/{distributed => federated}/cifar100.py (100%) rename fltk/datasets/{distributed => federated}/dataset.py (93%) rename fltk/datasets/{distributed => federated}/fashion_mnist.py (100%) rename fltk/datasets/{distributed => federated}/mnist.py (100%) delete mode 100644 fltk/datasets/loader_util.py delete mode 100644 fltk/util/generate_data_distribution.py delete mode 100644 fltk/util/generate_docker_compose.py delete mode 100644 fltk/util/generate_docker_compose_2.py delete mode 100644 fltk/util/generate_experiments.py delete mode 100644 fltk/util/offloading_estimate.py diff --git a/configs/distributed_tasks/example_arrival_config.json b/configs/distributed_tasks/example_arrival_config.json index a661b8dc..7f2735a1 100644 --- a/configs/distributed_tasks/example_arrival_config.json +++ b/configs/distributed_tasks/example_arrival_config.json @@ -1,58 +1,75 @@ -[ - { - "type": "distributed", - "jobClassParameters": { - "networkConfiguration": { - "network": "FashionMNISTCNN", - "lossFunction": "CrossEntropyLoss", - "dataset": "mnist" - }, - "systemParameters": { - "dataParallelism": 2, - "configurations": { - "default": { - "cores": "1000m", - "memory": "1Gi" - } - } - }, - "hyperParameters": { - "default": { - "batchSize": 128, - "testBatchSize": 128, - "learningRateDecay": 0.0002, - "optimizerConfig": { - "type": "Adam", - "learningRate": 0.001, - "betas": [ - 0.9, - 0.999 - ] +{ + "experimentConfiguration": { + "randomSeed": [ + 1, + 41, + 42, + 43, + 430 + ] + }, + "trainTasks": [ + { + "type": "distributed", + "lambda": 0.004, + "preemptJobs": false, + "jobClassParameters": [ + { + "classProbability": 0.1, + "priorities": [ + { + "priority": 1, + "probability": 0.9 + }, + { + "priority": 0, + "probability": 0.1 + } + ], + "networkConfiguration": { + "network": "FashionMNISTCNN", + "lossFunction": "CrossEntropyLoss", + "dataset": "mnist" + }, + "systemParameters": { + "dataParallelism": 2, + "configurations": { + "default": { + "cores": "1000m", + "memory": "1Gi" + } + } + }, + "hyperParameters": { + "default": { + "totalEpochs": 100, + "batchSize": 128, + "testBatchSize": 128, + "learningRateDecay": 0.0002, + "optimizerConfig": { + "type": "Adam", + "learningRate": 0.001, + "betas": [ + 0.9, + 0.999 + ] + }, + "schedulerConfig": { + "schedulerStepSize": 50, + "schedulerGamma": 0.5, + "minimumLearningRate": 1e-10 + } + }, + "configurations": { + "Master": null, + "Worker": null + } }, - "schedulerConfig": { - "schedulerStepSize": 50, - "schedulerGamma": 0.5, - "minimumLearningRate": 1e-10 + "learningParameters": { + "cuda": false } - }, - "configurations": { - "Master": null, - "Worker": null } - }, - "learningParameters": { - "totalEpochs": 100, - "cuda": false - }, - "experimentConfiguration": { - "randomSeed": [ - 1, - 41, - 42, - 43, - 430 - ] - } + ] } - } -] \ No newline at end of file + ] +} \ No newline at end of file diff --git a/configs/example_cloud_experiment.json b/configs/example_cloud_experiment.json index b6055ff6..d2518c6a 100644 --- a/configs/example_cloud_experiment.json +++ b/configs/example_cloud_experiment.json @@ -1,15 +1,14 @@ { "cluster": { "orchestrator": { - "wait_for_clients": true, - "service": "fl-server.test.svc.cluster.local", - "nic": "eth0" + "orchestrator_type": "simulated" }, "client": { "prefix": "client", "tensorboard_active": false }, - "image": "gcr.io/test-bed-distml/fltk:latest" + "image": "gcr.io/test-bed-distml/fltk:latest", + "namespace": "test" }, "execution_config": { "duration": 3600, diff --git a/fltk/datasets/distributed/__init__.py b/fltk/datasets/federated/__init__.py similarity index 100% rename from fltk/datasets/distributed/__init__.py rename to fltk/datasets/federated/__init__.py diff --git a/fltk/datasets/distributed/cifar10.py b/fltk/datasets/federated/cifar10.py similarity index 97% rename from fltk/datasets/distributed/cifar10.py rename to fltk/datasets/federated/cifar10.py index 11ae3e80..1d62e45d 100644 --- a/fltk/datasets/distributed/cifar10.py +++ b/fltk/datasets/federated/cifar10.py @@ -2,7 +2,7 @@ from torchvision import datasets from torchvision import transforms -from fltk.datasets.distributed.dataset import DistDataset +from fltk.datasets.distributed import DistDataset from fltk.samplers import get_sampler from fltk.util.config import FedLearningConfig diff --git a/fltk/datasets/distributed/cifar100.py b/fltk/datasets/federated/cifar100.py similarity index 100% rename from fltk/datasets/distributed/cifar100.py rename to fltk/datasets/federated/cifar100.py diff --git a/fltk/datasets/distributed/dataset.py b/fltk/datasets/federated/dataset.py similarity index 93% rename from fltk/datasets/distributed/dataset.py rename to fltk/datasets/federated/dataset.py index 276c3906..4a5cc56b 100644 --- a/fltk/datasets/distributed/dataset.py +++ b/fltk/datasets/federated/dataset.py @@ -1,7 +1,5 @@ from abc import abstractmethod -from typing import Any -# from fltk.util.arguments import Arguments from fltk.util.config import FedLearningConfig from fltk.util.log import getLogger diff --git a/fltk/datasets/distributed/fashion_mnist.py b/fltk/datasets/federated/fashion_mnist.py similarity index 100% rename from fltk/datasets/distributed/fashion_mnist.py rename to fltk/datasets/federated/fashion_mnist.py diff --git a/fltk/datasets/distributed/mnist.py b/fltk/datasets/federated/mnist.py similarity index 100% rename from fltk/datasets/distributed/mnist.py rename to fltk/datasets/federated/mnist.py diff --git a/fltk/datasets/loader_util.py b/fltk/datasets/loader_util.py deleted file mode 100644 index 29adc332..00000000 --- a/fltk/datasets/loader_util.py +++ /dev/null @@ -1,47 +0,0 @@ -from fltk.datasets.distributed import DistMNISTDataset, DistFashionMNISTDataset, DistCIFAR100Dataset, DistCIFAR10Dataset -from fltk.datasets import MNIST, FashionMNISTDataset, CIFAR10Dataset, CIFAR100Dataset -from fltk.util.config.definitions.dataset import Dataset - - -def available_datasets(): - return { - Dataset.cifar10: DistCIFAR10Dataset, - Dataset.cifar100: DistCIFAR100Dataset, - Dataset.fashion_mnist: DistFashionMNISTDataset, - Dataset.mnist: DistMNISTDataset - } - -def available_dataparallel_datasets(): - return { - Dataset.cifar10: CIFAR10Dataset, - Dataset.cifar100: CIFAR100Dataset, - Dataset.fashion_mnist: FashionMNISTDataset, - Dataset.mnist: MNIST - } - -def get_dataset(name: Dataset): - return available_datasets()[name] - - -def get_train_loader_path(name: Dataset) -> str: - paths = { - Dataset.cifar10: 'data_loaders/cifar10/train_data_loader.pickle', - Dataset.fashion_mnist: 'data_loaders/fashion-mnist/train_data_loader.pickle', - Dataset.cifar100: 'data_loaders/cifar100/train_data_loader.pickle', - Dataset.mnist: 'data_loaders/mnist/train_data_loader.pickle', - } - return paths[name] - - -def get_test_loader_path(name: Dataset) -> str: - paths = { - Dataset.cifar10: 'data_loaders/cifar10/test_data_loader.pickle', - Dataset.fashion_mnist: 'data_loaders/fashion-mnist/test_data_loader.pickle', - Dataset.cifar100: 'data_loaders/cifar100/test_data_loader.pickle', - Dataset.mnist: 'data_loaders/mnist/test_data_loader.pickle', - } - return paths[name] - - -def get_dist_dataset(name: Dataset): - return available_dataparallel_datasets()[name] \ No newline at end of file diff --git a/fltk/util/generate_data_distribution.py b/fltk/util/generate_data_distribution.py deleted file mode 100644 index 59272559..00000000 --- a/fltk/util/generate_data_distribution.py +++ /dev/null @@ -1,70 +0,0 @@ -import pathlib -import os -import logging - -from fltk.datasets.distributed import DistCIFAR10Dataset, DistCIFAR100Dataset, DistFashionMNISTDataset -# from fltk.datasets import CIFAR10Dataset, FashionMNISTDataset, CIFAR100Dataset -from fltk.util.data_loader_utils import generate_train_loader, generate_test_loader, save_data_loader_to_file - -logging.basicConfig(level=logging.DEBUG) - - -if __name__ == '__main__': - args = Arguments(logging) - - # --------------------------------- - # ------------ CIFAR10 ------------ - # --------------------------------- - dataset = DistCIFAR10Dataset(args) - TRAIN_DATA_LOADER_FILE_PATH = "data_loaders/cifar10/train_data_loader.pickle" - TEST_DATA_LOADER_FILE_PATH = "data_loaders/cifar10/test_data_loader.pickle" - - if not os.path.exists("data_loaders/cifar10"): - pathlib.Path("data_loaders/cifar10").mkdir(parents=True, exist_ok=True) - - train_data_loader = generate_train_loader(args, dataset) - test_data_loader = generate_test_loader(args, dataset) - - with open(TRAIN_DATA_LOADER_FILE_PATH, "wb") as f: - save_data_loader_to_file(train_data_loader, f) - - with open(TEST_DATA_LOADER_FILE_PATH, "wb") as f: - save_data_loader_to_file(test_data_loader, f) - - # --------------------------------- - # --------- Fashion-MNIST --------- - # --------------------------------- - dataset = DistFashionMNISTDataset(args) - TRAIN_DATA_LOADER_FILE_PATH = "data_loaders/fashion-mnist/train_data_loader.pickle" - TEST_DATA_LOADER_FILE_PATH = "data_loaders/fashion-mnist/test_data_loader.pickle" - - if not os.path.exists("data_loaders/fashion-mnist"): - pathlib.Path("data_loaders/fashion-mnist").mkdir(parents=True, exist_ok=True) - - train_data_loader = generate_train_loader(args, dataset) - test_data_loader = generate_test_loader(args, dataset) - - with open(TRAIN_DATA_LOADER_FILE_PATH, "wb") as f: - save_data_loader_to_file(train_data_loader, f) - - with open(TEST_DATA_LOADER_FILE_PATH, "wb") as f: - save_data_loader_to_file(test_data_loader, f) - - # --------------------------------- - # ------------ CIFAR100 ----------- - # --------------------------------- - dataset = DistCIFAR100Dataset(args) - TRAIN_DATA_LOADER_FILE_PATH = "data_loaders/cifar100/train_data_loader.pickle" - TEST_DATA_LOADER_FILE_PATH = "data_loaders/cifar100/test_data_loader.pickle" - - if not os.path.exists("data_loaders/cifar100"): - pathlib.Path("data_loaders/cifar100").mkdir(parents=True, exist_ok=True) - - train_data_loader = generate_train_loader(args, dataset) - test_data_loader = generate_test_loader(args, dataset) - - with open(TRAIN_DATA_LOADER_FILE_PATH, "wb") as f: - save_data_loader_to_file(train_data_loader, f) - - with open(TEST_DATA_LOADER_FILE_PATH, "wb") as f: - save_data_loader_to_file(test_data_loader, f) diff --git a/fltk/util/generate_docker_compose.py b/fltk/util/generate_docker_compose.py deleted file mode 100644 index 6f776c2b..00000000 --- a/fltk/util/generate_docker_compose.py +++ /dev/null @@ -1,574 +0,0 @@ -# pylint: disable=unused-variable,redefined-builtin -import argparse -import copy -from pathlib import Path - -import yaml - -# global_template_path = './deploy/templates' - -global_template_path = Path(__file__).absolute().parent.parent.parent / 'deploy' / 'templates' -global_template_path = global_template_path.__str__() -print(global_template_path) -def load_system_template(template_path = global_template_path): - print(f'Loading system template from {template_path}/system_stub.yml') - with open(f'{template_path}/system_stub.yml') as file: - documents = yaml.full_load(file) - return documents - -def load_client_template(type='default', template_path = global_template_path): - with open(f'{template_path}/client_stub_{type}.yml') as file: - documents = yaml.full_load(file) - return documents - -def get_deploy_path(name: str): - return f'{Path(global_template_path).parent}/{name}' - - -def generate_client(id, template: dict, world_size: int, type='default', cpu_set=''): - local_template = copy.deepcopy(template) - key_name = list(local_template.keys())[0] - container_name = f'client_{type}_{id}' - local_template[container_name] = local_template.pop(key_name) - for key, item in enumerate(local_template[container_name]['environment']): - if item == 'RANK={rank}': - local_template[container_name]['environment'][key] = item.format(rank=id) - if item == 'WORLD_SIZE={world_size}': - local_template[container_name]['environment'][key] = item.format(world_size=world_size) - # for key, item in enumerate(local_template[container_name]): - # if item == 'cpuset: {cpu_set}': - # local_template[container_name][key] = item.format(cpu_set=cpu_set) - - local_template[container_name]['ports'] = [f'{5000+id}:5000'] - local_template[container_name]['cpuset'] = f'{cpu_set}' - return local_template, container_name - -def generate_compose_file(): - print() - - -def generate_p30_freezing_effect_dev(): - template_path = get_deploy_path('p28_non_iid_effect') - num_clients = 6 - cpu_per_client = 1 - num_cpus = 20 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_set = 0 - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'default' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_p28_non_iid_effect(): - template_path = get_deploy_path('p28_non_iid_effect') - num_clients = 10 - cpu_per_client = 1 - num_cpus = 20 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_set = 0 - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'default' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_p23_freezoff_w9s3(): - template_path = get_deploy_path('p23_freezoff_w9s3') - num_clients = 9 - cpu_per_client = 1 - num_cpus = 20 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_set = 0 - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'default' - if 0 < client_id <= 3: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 3 < client_id <= 6: - client_type = 'medium' - cpu_set = f'{cpu_idx}-{cpu_idx+1}' - cpu_idx += 2 - elif 6 < client_id <= 9: - client_type = 'fast' - cpu_set = f'{cpu_idx}-{cpu_idx + 2}' - cpu_idx += 3 - else: - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_p23_freezoff_w9s3_half(): - template_path = get_deploy_path('p23_freezoff_w9s3-half') - num_clients = 9 - cpu_per_client = 1 - num_cpus = 20 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_set = 0 - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'default' - if 0 < client_id <= 3: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 3 < client_id <= 6: - client_type = 'medium' - cpu_set = f'{cpu_idx}-{cpu_idx+1}' - cpu_idx += 2 - elif 6 < client_id <= 9: - client_type = 'fast' - cpu_set = f'{cpu_idx}-{cpu_idx + 2}' - cpu_idx += 3 - else: - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_p23_freezoff_w9s3_fast(): - template_path = get_deploy_path('p23_freezoff_w9s3_fast') - num_clients = 9 - cpu_per_client = 1 - num_cpus = 20 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_set = 0 - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'default' - if 0 < client_id <= 3: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 3 < client_id <= 6: - client_type = 'medium' - cpu_set = f'{cpu_idx}-{cpu_idx+1}' - cpu_idx += 2 - elif 6 < client_id <= 9: - client_type = 'fast' - cpu_set = f'{cpu_idx}-{cpu_idx + 2}' - cpu_idx += 3 - else: - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - - - -def generate_terminate(num_clients = 16, medium=False): - template_path = get_deploy_path('terminate') - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - if client_id < 5: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - else: - client_type = 'medium' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_dev(num_clients = 2, medium=False): - template_path = get_deploy_path('dev') - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - if not medium: - client_type = 'fast' - cpu_set = f'{cpu_idx}-{cpu_idx + 2}' - cpu_idx += 3 - else: - client_type = 'medium' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_p13_w6(): - template_path = get_deploy_path('p11_freezoff') - num_clients= 6 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'default' - if 0 < client_id <= 2: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 2 < client_id <= 4: - client_type = 'medium' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 4 < client_id <= 6: - client_type = 'fast' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - else: - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - - -def generate_check_w4(): - template_path = get_deploy_path('p11_freezoff') - num_clients= 4 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'fast' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_check_w18(): - template_path = get_deploy_path('p11_freezoff') - num_clients= 18 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'fast' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - - -def generate_check_w18_fast(): - template_path = get_deploy_path('p11_freezoff_fast') - num_clients= 6 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'fast' - cpu_set = f'{cpu_idx}-{cpu_idx + 2}' - cpu_idx += 3 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - - -def generate_p11_freezoff(): - template_path = get_deploy_path('p11_freezoff') - num_clients= 18 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'default' - if 0 < client_id <= 6: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 6 < client_id <= 12: - client_type = 'medium' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 12 < client_id <= 18: - client_type = 'fast' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - else: - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - - -def generate_tifl_15(): - template_path = get_deploy_path('tifl-15') - num_clients= 18 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 2 - for client_id in range(1, num_clients + 1): - client_type = 'default' - if 0 < client_id <= 6: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 6 < client_id <= 12: - client_type = 'medium' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 12 < client_id <= 18: - client_type = 'fast' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - else: - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - - -def generate_tifl_3(): - template_path = get_deploy_path('tifl-15') - num_clients= 3 - world_size = num_clients + 1 - system_template: dict = load_system_template(template_path=template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_idx = 3 - for client_id in range(1, num_clients + 1): - client_type = 'default' - if 0 < client_id <= 1: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 1 < client_id <= 2: - client_type = 'medium' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif 2 < client_id <= 3: - client_type = 'fast' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - else: - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type, template_path=template_path) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, - cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_offload_exp(): - num_clients = 4 - cpu_per_client = 1 - num_cpus = 20 - world_size = num_clients + 1 - system_template: dict = load_system_template() - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - cpu_set = 0 - cpu_idx = 3 - for client_id in range(1, num_clients + 1): - client_type = 'medium' - client_type = 'default' - if client_id == 1 or client_id == 2: - client_type = 'medium' - cpu_set = f'{cpu_idx}-{cpu_idx+1}' - cpu_idx += 2 - elif client_id == 3: - client_type = 'slow' - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - elif client_id == 4: - client_type = 'fast' - cpu_set = f'{cpu_idx}-{cpu_idx + 2}' - cpu_idx += 3 - else: - cpu_set = f'{cpu_idx}' - cpu_idx += 1 - - client_template: dict = load_client_template(type=client_type) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type, cpu_set=cpu_set) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate(num_clients: int): - world_size = num_clients + 1 - system_template :dict = load_system_template() - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - - for client_id in range(1, num_clients+1): - client_type = 'default' - if client_id == 1: - client_type='slow' - if client_id == 2: - client_type='medium' - client_template: dict = load_client_template(type=client_type) - client_definition, container_name = generate_client(client_id, client_template, world_size, type=client_type) - system_template['services'].update(client_definition) - - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def run(name, num_clients = None, medium=False): - exp_dict = { - 'tifl-15': generate_tifl_15, - 'dev': generate_dev, - 'terminate': generate_terminate, - 'p11_freezoff': generate_p11_freezoff, - 'p13_w6' : generate_p13_w6, - 'p23_w9s3': generate_p23_freezoff_w9s3, - 'p23_w9s3-half': generate_p23_freezoff_w9s3_half, - 'p23_w9s3_fast': generate_p23_freezoff_w9s3_fast, - 'p28_non_iid_effect': generate_p28_non_iid_effect, - 'p30_dev': generate_p30_freezing_effect_dev, - 'generate_check_w4': generate_check_w4, - 'generate_check_w18': generate_check_w18, - 'generate_check_w18_fast': generate_check_w18_fast - } - if num_clients: - exp_dict[name](num_clients, medium) - else: - exp_dict[name]() - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generate docker-compose file') - parser.add_argument('name', type=str, - help='Name of an experiment') - parser.add_argument('--clients', type=int, help='Set the number of clients in the system', default=None) - args = parser.parse_args() - run(args.name, args.clients) - print('Done') - diff --git a/fltk/util/generate_docker_compose_2.py b/fltk/util/generate_docker_compose_2.py deleted file mode 100644 index 57e24e6c..00000000 --- a/fltk/util/generate_docker_compose_2.py +++ /dev/null @@ -1,136 +0,0 @@ -import argparse -import copy -from pathlib import Path -import yaml -import numpy as np - - -def load_yaml_file(file_path: Path): - with open(file_path) as file: - return yaml.full_load(file) - - -def generate_client(identifier, template: dict, world_size: int, tpe='default', cpu_set=None, num_cpus=1): - local_template = copy.deepcopy(template) - key_name = list(local_template.keys())[0] - container_name = f'client_{tpe}_{identifier}' - local_template[container_name] = local_template.pop(key_name) - for key, item in enumerate(local_template[container_name]['environment']): - if item == 'RANK={rank}': - local_template[container_name]['environment'][key] = item.format(rank=identifier) - if item == 'WORLD_SIZE={world_size}': - local_template[container_name]['environment'][key] = item.format(world_size=world_size) - local_template[container_name]['ports'] = [f'{5000 + identifier}:5000'] - if cpu_set: - local_template[container_name]['cpuset'] = f'{cpu_set}' - else: - local_template[container_name].pop('cpuset') - local_template[container_name]['deploy']['resources']['limits']['cpus'] = f'{num_cpus}' - return local_template, container_name - - -def gen_client(name: str, client_dict: dict, base_path: Path): - """ - rank (id) - num_cpu - cpu_set - name - """ - client_descr_template = { - 'rank': 0, - 'num_cpu': 1, - 'num_cores': None, - 'name': name, - 'stub-file': 'stub.yml' - } - print(Path.cwd()) - mu = client_dict['cpu-speed'] - sigma = client_dict['cpu-variation'] - n = client_dict['amount'] - np.random.seed(0) - stub_file = base_path / client_dict['stub-name'] - stub_data = load_yaml_file(stub_file) - if client_dict['pin-cores'] is True: - client_descr_template['num_cores'] = client_dict['num-cores'] - client_descr_template['stub-file'] = client_dict['stub-name'] - client_cpu_speeds = np.abs(np.round(np.random.normal(mu, sigma, size=n), 2)) - client_descriptions = [] - for cpu_speed in client_cpu_speeds: - client_descr = copy.deepcopy(client_descr_template) - client_descr['num_cpu'] = cpu_speed - client_descriptions.append(client_descr) - return client_descriptions - - -def generate_clients_proporties(clients_dict: dict, file_path: Path): - results = [] - for k,v in clients_dict.items(): - results += gen_client(k, v, file_path) - return results - -def generate_compose_file_from_dict(system: dict): - file_path = Path(system['base_path']) - client_descriptions = generate_clients_proporties(system['clients'], file_path) - last_core_id = 0 - world_size = len(client_descriptions) + 1 - system_template_path = file_path / 'system_stub.yml' - - system_template: dict = load_yaml_file(system_template_path) - - for key, item in enumerate(system_template['services']['fl_server']['environment']): - if item == 'WORLD_SIZE={world_size}': - system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) - if system['federator']['pin-cores']: - cpu_set: str - amount = system['federator']['num-cores'] - if amount > 1: - cpu_set = f'{last_core_id}-{last_core_id + amount - 1}' - else: - cpu_set = f'{last_core_id}' - system_template['services']['fl_server']['cpuset'] = cpu_set - last_core_id += amount - else: - system_template['services']['fl_server'].pop('cpuset') - for idx, client_d in enumerate(client_descriptions): - stub_file = file_path / client_d['stub-file'] - stub_data = load_yaml_file(stub_file) - cpu_set = None - if client_d['num_cores']: - amount = client_d['num_cores'] - if amount > 1: - cpu_set = f'{last_core_id}-{last_core_id + amount - 1}' - else: - cpu_set = f'{last_core_id}' - last_core_id += amount - local_template, container_name = generate_client(idx + 1, stub_data, world_size, client_d['name'], cpu_set, - client_d['num_cpu']) - system_template['services'].update(local_template) - print(container_name) - with open(r'./docker-compose.yml', 'w') as file: - yaml.dump(system_template, file, sort_keys=False) - -def generate_compose_file(path: Path): - """ - Used properties: - - World size - - num clients? - - path to deploy files - - random seed? - """ - - system_path = path / 'description.yml' - system = load_yaml_file(system_path) - # path = Path('deploy/dev_generate') - generate_compose_file_from_dict(system) - - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generate docker-compose file') - parser.add_argument('path', type=str, - help='Path to a deployment config folder') - parser.add_argument('--clients', type=int, help='Set the number of clients in the system', default=None) - args = parser.parse_args() - path = Path(args.path) - generate_compose_file(path) - print('done') \ No newline at end of file diff --git a/fltk/util/generate_experiments.py b/fltk/util/generate_experiments.py deleted file mode 100644 index 9cb49847..00000000 --- a/fltk/util/generate_experiments.py +++ /dev/null @@ -1,168 +0,0 @@ -import copy -from pathlib import Path -import os -import yaml -from fltk.util.generate_docker_compose_2 import generate_compose_file_from_dict - - -def rm_tree(pth: Path): - for child in pth.iterdir(): - if child.is_file(): - child.unlink() - # else: - # rm_tree(child) - # pth.rmdir() - - -def check_num_clients_consistency(cfg_data: dict): - if type(cfg_data) is str: - cfg_data = yaml.safe_load(copy.deepcopy(cfg_data)) - - if 'deploy' in cfg_data and 'docker' in cfg_data['deploy']: - num_docker_clients = sum([x['amount'] for x in cfg_data['deploy']['docker']['clients'].values()]) - if cfg_data['num_clients'] != num_docker_clients: - print('[Warning]\t Number of docker clients is not equal to the num_clients property!') - - -def generate(base_path: Path, config_path: Path, **kwargs): - descr_path = base_path / 'descr.yaml' - - exp_cfg_list = [x for x in base_path.iterdir() if '.cfg' in x.suffixes] - descr_data = '' - with open(descr_path) as descr_f: - descr_data = descr_f.read() - exps_path = base_path / 'exps' - rm_tree(exps_path) - exps_path.mkdir(parents=True, exist_ok=True) - - check_num_clients_consistency(descr_data) - for exp_cfg in exp_cfg_list: - exp_cfg_data = '' - with open(exp_cfg) as exp_f: - exp_cfg_data = exp_f.read() - - exp_data = descr_data + exp_cfg_data - exp_data += f'\nexperiment_prefix: \'{base_path.name}_{exp_cfg.name.split(".")[0]}\'\n' - filename = '.'.join([exp_cfg.name.split('.')[0], exp_cfg.name.split('.')[2]]) - with open(exps_path / filename, mode='w') as f: - f.write(exp_data) - print('Done') - - -# def run(): -# base_path = Path(__file__).parent -# descr_path = base_path / 'descr.yaml' -# -# exp_cfg_list = [x for x in base_path.iterdir() if '.cfg' in x.suffixes] -# descr_data = '' -# with open(descr_path) as descr_f: -# descr_data = descr_f.read() -# -# exps_path = base_path / 'exps' -# exps_path.mkdir(parents=True, exist_ok=True) -# for exp_cfg in exp_cfg_list: -# exp_cfg_data = '' -# replications = 1 -# with open(exp_cfg) as exp_f: -# exp_cfg_data = exp_f.read() -# for replication_id in range(replications): -# exp_data = descr_data + exp_cfg_data -# exp_data += f'\nexperiment_prefix: \'{Path(__file__).parent.name}_{exp_cfg.name.split(".")[0]}\'\n' -# filename = '.'.join([exp_cfg.name.split('.')[0], exp_cfg.name.split('.')[2]]) -# with open(exps_path / filename, mode='w') as f: -# f.write(exp_data) -# print('Done') - - -def run(base_path: Path, config_path: Path, **kwargs): - print(f'Run {base_path}') - print(list(base_path.iterdir())) - descr_path = base_path / 'descr.yaml' - exp_cfg_list = [x for x in base_path.iterdir() if '.cfg' in x.suffixes] - descr_data = '' - with open(descr_path) as descr_f: - descr_data = yaml.safe_load(descr_f.read()) - - replications = 1 - if 'replications' in descr_data: - replications = descr_data['replications'] - run_docker = False - if 'deploy' in descr_data and 'docker' in descr_data['deploy']: - # if 'docker_system' in descr_data: - # Run in docker - # Generate Docker - print(descr_data) - docker_deploy_path = Path(descr_data['deploy']['docker']['base_path']) - - print(docker_deploy_path) - run_docker = True - generate_compose_file_from_dict(descr_data['deploy']['docker']) - # generate_compose_file(docker_deploy_path) - - exp_files = [x for x in (base_path / 'exps').iterdir() if x.suffix in ['.yaml', '.yml']] - - cmd_list = [] - print(exp_files) - if run_docker: - first_prefix = '--build' - for exp_cfg_file in exp_files: - for replication_id in range(replications): - cmd = f'export OPTIONAL_PARAMS="--prefix={replication_id}";export EXP_CONFIG_FILE="{exp_cfg_file}"; docker-compose --compatibility up {first_prefix};' - cmd_list.append(cmd) - # print(f'Running cmd: "{cmd}"') - # os.system(cmd) - first_prefix = '' - else: - print('Switching to direct mode') - for exp_cfg_file in exp_files: - for replication_id in range(replications): - # cmd = f'export OPTIONAL_PARAMS="--prefix={replication_id}";export EXP_CONFIG_FILE="{exp_cfg_file}"; docker-compose --compatibility up {first_prefix};' - cmd = f'python3 -m fltk single {exp_cfg_file} --prefix={replication_id}' - cmd_list.append(cmd) - - for cmd in cmd_list: - print(f'Running cmd: "{cmd}"') - os.system(cmd) - print('Done') - # docker_system - - - # name = 'dev' - # generate_docker(name) - # base_path = f'{Path(__file__).parent}' - # exp_list = [ - # 'fedavg.yaml', - # ] - # exp_list = [f'{base_path}/exps/{x}' for x in exp_list] - # first_prefix = '--build' - # for exp_cfg_file in exp_list: - # cmd = f'export EXP_CONFIG_FILE="{exp_cfg_file}"; docker-compose --compatibility up {first_prefix};' - # print(f'Running cmd: "{cmd}"') - # os.system(cmd) - # first_prefix = '' - - # print('Done') - -# if __name__ == '__main__': -# base_path = Path(__file__).parent -# descr_path = base_path / 'descr.yaml' -# -# exp_cfg_list = [x for x in base_path.iterdir() if '.cfg' in x.suffixes] -# descr_data = '' -# with open(descr_path) as descr_f: -# descr_data = descr_f.read() -# exps_path = base_path / 'exps' -# exps_path.mkdir(parents=True, exist_ok=True) -# for exp_cfg in exp_cfg_list: -# exp_cfg_data = '' -# with open(exp_cfg) as exp_f: -# exp_cfg_data = exp_f.read() -# -# exp_data = descr_data + exp_cfg_data -# exp_data += f'\nexperiment_prefix: \'{Path(__file__).parent.name}_{exp_cfg.name.split(".")[0]}\'\n' -# filename = '.'.join([exp_cfg.name.split('.')[0], exp_cfg.name.split('.')[2]]) -# with open(exps_path / filename, mode='w') as f: -# f.write(exp_data) -# print('Done') -# -# diff --git a/fltk/util/offloading_estimate.py b/fltk/util/offloading_estimate.py deleted file mode 100644 index cfbf25de..00000000 --- a/fltk/util/offloading_estimate.py +++ /dev/null @@ -1,61 +0,0 @@ - -def calc_optimal_offloading_point(profiler_data, time_till_deadline, iterations_left): - ff, cf, cb, fb = profiler_data - full_network = ff + cf + cb + fb - frozen_network = ff + cf + cb - split_point = 0 - for z in range(iterations_left, -1, -1): - x = z - y = iterations_left - x - # print(z) - new_est_split = (x * full_network) + (y * frozen_network) - split_point = x - if new_est_split < time_till_deadline: - break - - -def estimate(): - """ - freeze_network = ff + cf + cb + fb - frozen_network = ff + cf + cb - - td = time until deadline - cl = cycles left - - a = 1 - b = cl - a - - - """ - np = { - 'a': 2, - 'b': 1, - 'c': 3, - 'd': 4, - } - - sp = { - 'time_left': 400, - 'iter_left': 44 - } - - f_n = np['a'] + np['b'] + np['c'] + np['d'] - o_n = np['a'] + np['b'] + np['c'] - est_full_comp_time = f_n * sp['iter_left'] - new_est = o_n * sp['iter_left'] - x = 20 - y = sp['iter_left'] - x - new_est_split = (x * f_n) + (y * o_n) - - print(f'estimate: {est_full_comp_time} < {sp["time_left"]} ? {est_full_comp_time