From 849021f1472b35a22d2c3f39bc3a2cce49cc25b3 Mon Sep 17 00:00:00 2001 From: bacox Date: Fri, 11 Mar 2022 16:27:07 +0100 Subject: [PATCH] Add systes generators --- configs/dev_mnist/descr.yaml | 1 + configs/dev_mnist/exps/fedavg.yaml | 9 +- configs/dev_mnist/exps/fedavg_direct.yaml | 38 +++++ configs/dev_mnist/fedavg.cfg.yaml | 4 +- configs/dev_mnist/fedavg_direct.cfg.yaml | 5 + configs/dev_mnist/gen.py | 9 +- configs/dev_mnist/run.py | 2 +- configs/dev_mnist_all/exps/fedavg.yaml | 1 + deploy/dev/client_stub_default.yml | 2 + deploy/dev/client_stub_fast.yml | 2 + deploy/dev/client_stub_medium.yml | 2 + deploy/dev/client_stub_slow.yml | 2 + deploy/dev/system_stub.yml | 2 + deploy/dev_generate/client_stub_medium.yml | 26 ++++ deploy/dev_generate/description.yml | 19 +++ deploy/dev_generate/stub_default.yml | 26 ++++ deploy/dev_generate/stub_fast.yml | 25 ++++ deploy/dev_generate/system_stub.yml | 29 ++++ fltk/__main__.py | 35 +++-- fltk/core/federator.py | 28 ++-- fltk/util/generate_docker_compose_2.py | 164 +++++++++++++++++++++ fltk/util/generate_experiments.py | 47 ++++++ 22 files changed, 444 insertions(+), 34 deletions(-) create mode 100644 configs/dev_mnist/exps/fedavg_direct.yaml create mode 100644 configs/dev_mnist/fedavg_direct.cfg.yaml create mode 100644 deploy/dev_generate/client_stub_medium.yml create mode 100644 deploy/dev_generate/description.yml create mode 100644 deploy/dev_generate/stub_default.yml create mode 100644 deploy/dev_generate/stub_fast.yml create mode 100644 deploy/dev_generate/system_stub.yml create mode 100644 fltk/util/generate_docker_compose_2.py create mode 100644 fltk/util/generate_experiments.py diff --git a/configs/dev_mnist/descr.yaml b/configs/dev_mnist/descr.yaml index d4357836..87b954ba 100644 --- a/configs/dev_mnist/descr.yaml +++ b/configs/dev_mnist/descr.yaml @@ -29,3 +29,4 @@ system: # nic: 'enp3s0' clients: amount: 2 +num_clients: 2 diff --git a/configs/dev_mnist/exps/fedavg.yaml b/configs/dev_mnist/exps/fedavg.yaml index 4c7ef222..89dc8a37 100644 --- a/configs/dev_mnist/exps/fedavg.yaml +++ b/configs/dev_mnist/exps/fedavg.yaml @@ -11,7 +11,7 @@ profiling_time: 100 warmup_round: false output_location: 'output/dev_p2' tensor_board_active: true -clients_per_round: 2 +clients_per_round: 4 node_groups: slow: [1, 1] medium: [2, 2] @@ -28,8 +28,11 @@ system: nic: 'eth0' # nic: 'enp3s0' clients: - amount: 2 + amount: 4 +num_clients: 4 # Individual configuration offload_stategy: vanilla deadline: 500 -experiment_prefix: 'dev_mnist_fedavg' +single_machine: false +real_time: true +experiment_prefix: 'util_fedavg' diff --git a/configs/dev_mnist/exps/fedavg_direct.yaml b/configs/dev_mnist/exps/fedavg_direct.yaml new file mode 100644 index 00000000..ab294bb3 --- /dev/null +++ b/configs/dev_mnist/exps/fedavg_direct.yaml @@ -0,0 +1,38 @@ +--- +# Experiment configuration +total_epochs: 3 +epochs_per_cycle: 1 +wait_for_clients: true +net: MNISTCNN +dataset: mnist +# Use cuda is available; setting to false will force CPU +cuda: false +profiling_time: 100 +warmup_round: false +output_location: 'output/dev_p2' +tensor_board_active: true +clients_per_round: 2 +node_groups: + slow: [1, 1] + medium: [2, 2] + fast: [3, 3] +sampler: "uniform" # "limit labels" || "q sampler" || "dirichlet" || "uniform" (default) +#sampler: "uniform" # "limit labels" || "q sampler" || "dirichlet" || "uniform" (default) +sampler_args: + - 0.07 # label limit || q probability || alpha || unused + - 42 # random seed || random seed || random seed || unused +system: + federator: + # hostname: '131.180.203.94' + hostname: '10.5.0.11' + nic: 'eth0' + # nic: 'enp3s0' + clients: + amount: 2 +num_clients: 2 +# Individual configuration +offload_stategy: vanilla +deadline: 500 +single_machine: true +real_time: false +experiment_prefix: 'util_fedavg_direct' diff --git a/configs/dev_mnist/fedavg.cfg.yaml b/configs/dev_mnist/fedavg.cfg.yaml index ecb5bc3e..17bd81b1 100644 --- a/configs/dev_mnist/fedavg.cfg.yaml +++ b/configs/dev_mnist/fedavg.cfg.yaml @@ -1,3 +1,5 @@ # Individual configuration offload_stategy: vanilla -deadline: 500 \ No newline at end of file +deadline: 500 +single_machine: false +real_time: true \ No newline at end of file diff --git a/configs/dev_mnist/fedavg_direct.cfg.yaml b/configs/dev_mnist/fedavg_direct.cfg.yaml new file mode 100644 index 00000000..25a64bda --- /dev/null +++ b/configs/dev_mnist/fedavg_direct.cfg.yaml @@ -0,0 +1,5 @@ +# Individual configuration +offload_stategy: vanilla +deadline: 500 +single_machine: true +real_time: false \ No newline at end of file diff --git a/configs/dev_mnist/gen.py b/configs/dev_mnist/gen.py index 168833f0..f268dc45 100644 --- a/configs/dev_mnist/gen.py +++ b/configs/dev_mnist/gen.py @@ -1,15 +1,14 @@ from pathlib import Path if __name__ == '__main__': - base_path = f'configs/{Path(__file__).parent.name}' - path = Path(base_path) - descr_path = path / 'descr.yaml' + base_path = Path(__file__).parent + descr_path = base_path / 'descr.yaml' - exp_cfg_list = [x for x in path.iterdir() if '.cfg' in x.suffixes] + exp_cfg_list = [x for x in base_path.iterdir() if '.cfg' in x.suffixes] descr_data = '' with open(descr_path) as descr_f: descr_data = descr_f.read() - exps_path = path / 'exps' + exps_path = base_path / 'exps' exps_path.mkdir(parents=True, exist_ok=True) for exp_cfg in exp_cfg_list: exp_cfg_data = '' diff --git a/configs/dev_mnist/run.py b/configs/dev_mnist/run.py index 8e1fad76..3714a567 100644 --- a/configs/dev_mnist/run.py +++ b/configs/dev_mnist/run.py @@ -5,7 +5,7 @@ if __name__ == '__main__': name = 'dev' generate_docker(name) - base_path = f'configs/{Path(__file__).parent.name}' + base_path = f'{Path(__file__).parent}' exp_list = [ 'fedavg.yaml', ] diff --git a/configs/dev_mnist_all/exps/fedavg.yaml b/configs/dev_mnist_all/exps/fedavg.yaml index 5e90d7fe..391bf17c 100644 --- a/configs/dev_mnist_all/exps/fedavg.yaml +++ b/configs/dev_mnist_all/exps/fedavg.yaml @@ -33,3 +33,4 @@ system: offload_stategy: vanilla deadline: 500000 experiment_prefix: 'dev_mnist_all_fedavg' +single_machine: false diff --git a/deploy/dev/client_stub_default.yml b/deploy/dev/client_stub_default.yml index 3a1774cf..e8e5b9ba 100644 --- a/deploy/dev/client_stub_default.yml +++ b/deploy/dev/client_stub_default.yml @@ -14,6 +14,8 @@ client_name: # name can be anything - RANK={rank} - WORLD_SIZE={world_size} - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 ports: - "5002:5000" # {machine-port}:{docker-port} depends_on: diff --git a/deploy/dev/client_stub_fast.yml b/deploy/dev/client_stub_fast.yml index f03012ff..4e8d2d78 100644 --- a/deploy/dev/client_stub_fast.yml +++ b/deploy/dev/client_stub_fast.yml @@ -14,6 +14,8 @@ client_name: # name can be anything - RANK={rank} - WORLD_SIZE={world_size} - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 ports: - "5002:5000" # {machine-port}:{docker-port} depends_on: diff --git a/deploy/dev/client_stub_medium.yml b/deploy/dev/client_stub_medium.yml index 49abdeb2..9d096797 100644 --- a/deploy/dev/client_stub_medium.yml +++ b/deploy/dev/client_stub_medium.yml @@ -14,6 +14,8 @@ client_name: # name can be anything - RANK={rank} - WORLD_SIZE={world_size} - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 ports: - "5002:5000" # {machine-port}:{docker-port} depends_on: diff --git a/deploy/dev/client_stub_slow.yml b/deploy/dev/client_stub_slow.yml index 9cbdabb5..deb37f37 100644 --- a/deploy/dev/client_stub_slow.yml +++ b/deploy/dev/client_stub_slow.yml @@ -14,6 +14,8 @@ client_name: # name can be anything - RANK={rank} - WORLD_SIZE={world_size} - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 ports: - "5002:5000" # {machine-port}:{docker-port} depends_on: diff --git a/deploy/dev/system_stub.yml b/deploy/dev/system_stub.yml index c84b2ecb..37404525 100644 --- a/deploy/dev/system_stub.yml +++ b/deploy/dev/system_stub.yml @@ -16,6 +16,8 @@ services: - RANK=0 - WORLD_SIZE={world_size} - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 ports: - "5000:5000" # {machine-port}:{docker-port} networks: diff --git a/deploy/dev_generate/client_stub_medium.yml b/deploy/dev_generate/client_stub_medium.yml new file mode 100644 index 00000000..9d096797 --- /dev/null +++ b/deploy/dev_generate/client_stub_medium.yml @@ -0,0 +1,26 @@ +client_name: # name can be anything +# container_name: federation-lab-client2 # what the name for this container would be + cpuset: {cpu_set} + restart: "no" # if it crashes for example + build: . # look for the docker file where this file is currently located + volumes: +# - ./docker_data:/opt/federation-lab/data + - ./data:/opt/federation-lab/data + - ./default_models:/opt/federation-lab/default_models + - ./data_loaders:/opt/federation-lab/data_loaders + - ./fltk:/opt/federation-lab/fltk + environment: + - PYTHONUNBUFFERED=1 + - RANK={rank} + - WORLD_SIZE={world_size} + - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 + ports: + - "5002:5000" # {machine-port}:{docker-port} + depends_on: + - "fl_server" + deploy: + resources: + limits: + cpus: '2' diff --git a/deploy/dev_generate/description.yml b/deploy/dev_generate/description.yml new file mode 100644 index 00000000..4a6f45b6 --- /dev/null +++ b/deploy/dev_generate/description.yml @@ -0,0 +1,19 @@ +federator: + stub-name: system_stub.yml + pin-cores: true + num-cores: 1 +clients: + fast: + stub-name: stub_default.yml + amount: 20 + pin-cores: true + num-cores: 1 + cpu-speed: 0.5 + cpu-variation: 0.16 + slow: + stub-name: stub_default.yml + amount: 0 + pin-cores: true + num-cores: 1 + cpu-speed: 1 + cpu-variation: 0 diff --git a/deploy/dev_generate/stub_default.yml b/deploy/dev_generate/stub_default.yml new file mode 100644 index 00000000..bc437b1c --- /dev/null +++ b/deploy/dev_generate/stub_default.yml @@ -0,0 +1,26 @@ +client_name: # name can be anything +# container_name: federation-lab-client2 # what the name for this container would be + cpuset: '{cpu_set}' + restart: "no" # if it crashes for example + build: . # look for the docker file where this file is currently located + volumes: + - ./data:/opt/federation-lab/data +# - ./docker_data:/opt/federation-lab/data + - ./default_models:/opt/federation-lab/default_models + - ./data_loaders:/opt/federation-lab/data_loaders + - ./fltk:/opt/federation-lab/fltk + environment: + - PYTHONUNBUFFERED=1 + - RANK={rank} + - WORLD_SIZE={world_size} + - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 + ports: + - "5002:5000" # {machine-port}:{docker-port} + depends_on: + - "fl_server" + deploy: + resources: + limits: + cpus: '{num_cpus}' \ No newline at end of file diff --git a/deploy/dev_generate/stub_fast.yml b/deploy/dev_generate/stub_fast.yml new file mode 100644 index 00000000..3b4aee9c --- /dev/null +++ b/deploy/dev_generate/stub_fast.yml @@ -0,0 +1,25 @@ +client_name: # name can be anything + cpuset: {cpu_set} + restart: "no" # if it crashes for example + build: . # look for the docker file where this file is currently located + volumes: +# - ./docker_data:/opt/federation-lab/data + - ./data:/opt/federation-lab/data + - ./default_models:/opt/federation-lab/default_models + - ./data_loaders:/opt/federation-lab/data_loaders + - ./fltk:/opt/federation-lab/fltk + environment: + - PYTHONUNBUFFERED=1 + - RANK={rank} + - WORLD_SIZE={world_size} + - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 + ports: + - "5002:5000" # {machine-port}:{docker-port} + depends_on: + - "fl_server" + deploy: + resources: + limits: + cpus: {num_cpus} diff --git a/deploy/dev_generate/system_stub.yml b/deploy/dev_generate/system_stub.yml new file mode 100644 index 00000000..37404525 --- /dev/null +++ b/deploy/dev_generate/system_stub.yml @@ -0,0 +1,29 @@ +# creating a multi-container docker +version: "3.3" +services: + fl_server: # name can be anything + container_name: federation-lab-server # what the name for this container would be + cpuset: '0-2' + restart: "no" # if it crashes for example + build: . # look for the docker file where this file is currently located + volumes: +# - ./data/MNIST:/opt/federation-lab/data/MNIST + - ./data:/opt/federation-lab/data + - ./output:/opt/federation-lab/output + - ./fltk:/opt/federation-lab/fltk + environment: + - PYTHONUNBUFFERED=1 + - RANK=0 + - WORLD_SIZE={world_size} + - EXP_CONFIG=${EXP_CONFIG_FILE} + - MASTER_HOSTNAME=10.5.0.11 + - NIC=eth0 + ports: + - "5000:5000" # {machine-port}:{docker-port} + networks: + default: + ipv4_address: 10.5.0.11 +networks: + default: + external: + name: local_network_dev \ No newline at end of file diff --git a/fltk/__main__.py b/fltk/__main__.py index cf614012..8a6e8990 100644 --- a/fltk/__main__.py +++ b/fltk/__main__.py @@ -93,9 +93,9 @@ from fltk.core.client import Client -print(sys.path) +# print(sys.path) # from fltk.core.federator import Federator as Fed -print(list(Path.cwd().iterdir())) +# print(list(Path.cwd().iterdir())) import argparse from enum import Enum from pathlib import Path @@ -103,6 +103,8 @@ from fltk.core.federator import Federator from fltk.util.config import Config from fltk.util.definitions import Aggregations, Optimizations +from fltk.util.generate_experiments import generate + def run_single(config_path: Path): @@ -152,7 +154,7 @@ def run_remote(config_path: Path, rank: int, nic=None, host=None): init_method=f'tcp://{os.environ["MASTER_ADDR"]}:{os.environ["MASTER_PORT"]}' ) if rank != 0: - print(f'Starting worker {rank}') + print(f'Starting worker {rank} with world size={config.world_size}') rpc.init_rpc( f"client{rank}", rank=rank, @@ -194,10 +196,20 @@ def add_default_arguments(parser): parser = argparse.ArgumentParser(prog='fltk', description='Experiment launcher for the Federated Learning Testbed (fltk)') subparsers = parser.add_subparsers(dest="action", required=True) - launch_parser = subparsers.add_parser('launch-util') + util_docker_parser = subparsers.add_parser('util-docker') + util_docker_parser.add_argument('name', type=str) + util_docker_parser.add_argument('--clients', type=int) + util_generate_parser = subparsers.add_parser('util-generate') + util_generate_parser.add_argument('path', type=str) + util_run_parser = subparsers.add_parser('util-run') + util_run_parser.add_argument('path', type=str) + + # launch_parser.add_argument('action', choices=['docker', 'generate', 'run']) + # launch_parser.add_argument('path', help='path or key') + remote_parser = subparsers.add_parser('remote') single_machine_parser = subparsers.add_parser('single') - add_default_arguments(launch_parser) + # add_default_arguments(launch_parser) add_default_arguments(remote_parser) add_default_arguments(single_machine_parser) @@ -211,10 +223,15 @@ def add_default_arguments(parser): # util_parser.add_argument('action') # print(sys.argv) args = parser.parse_args() - if args.action == 'launch-util': - pass - # run_single(Path(args.config)) - if args.action == 'remote': + if args.action == 'util-docker': + print('docker') + elif args.action == 'util-generate': + path = Path(args.path) + print(f'generate for {path}') + generate(path) + elif args.action == 'util-run': + print('run') # run_single(Path(args.config)) + elif args.action == 'remote': run_remote(Path(args.config), args.rank, args.nic, args.host) else: # Run single machine mode diff --git a/fltk/core/federator.py b/fltk/core/federator.py index 43d9c392..8151d8a7 100644 --- a/fltk/core/federator.py +++ b/fltk/core/federator.py @@ -25,6 +25,8 @@ class LocalClient: exp_data: DataContainer +def cb_factory(future: torch.Future, method, *args, **kwargs): + future.then(lambda x: method(x, *args, **kwargs)) class Federator(Node): clients: List[LocalClient] = [] @@ -190,27 +192,23 @@ def exec_round(self, id: int): training_futures: List[torch.Future] = [] + # def cb_factory(future: torch.Future, method, client, client_weights, client_sizes, num_epochs, name): + # future.then(lambda x: method(x, client, client_weights, client_sizes, num_epochs, client.name)) - def training_cb(fut: torch.Future, client: LocalClient): + def training_cb(fut: torch.Future, client_ref: LocalClient, client_weights, client_sizes, num_epochs): train_loss, weights, accuracy, test_loss, round_duration, train_duration, test_duration = fut.wait() - client_weights[client.name] = weights - client_data_size = self.message(client.ref, Client.get_client_datasize) - client_sizes[client.name] = client_data_size - self.logger.info(f'Training callback for client {client.name}') - client.exp_data.append( + self.logger.info(f'Training callback for client {client_ref.name} with accuracy={accuracy}') + client_weights[client_ref.name] = weights + client_data_size = self.message(client_ref.ref, Client.get_client_datasize) + client_sizes[client_ref.name] = client_data_size + client_ref.exp_data.append( ClientRecord(id, train_duration, test_duration, round_duration, num_epochs, 0, accuracy, train_loss, test_loss)) for client in selected_clients: - # future: torch.Future - # if not self.real_time: - # future = torch.futures.Future() - # future.set_result(self.message(client.ref, Client.exec_round, num_epochs)) - # future.then(lambda x: training_cb(x, client)) - # training_futures.append(future) - # else: future = self.message_async(client.ref, Client.exec_round, num_epochs) - future.then(lambda x: training_cb(x, client)) + cb_factory(future, training_cb, client, client_weights, client_sizes, num_epochs) + self.logger.info(f'Request sent to client {client.name}') training_futures.append(future) def all_futures_done(futures: List[torch.Future])->bool: @@ -221,7 +219,7 @@ def all_futures_done(futures: List[torch.Future])->bool: # self.logger.info(f'Waiting for other clients') self.logger.info(f'Continue with rest [1]') - + time.sleep(3) # for client in selected_clients: # # pbar.set_description(f'[Round {id:>3}] Running clients') diff --git a/fltk/util/generate_docker_compose_2.py b/fltk/util/generate_docker_compose_2.py new file mode 100644 index 00000000..a35bd43d --- /dev/null +++ b/fltk/util/generate_docker_compose_2.py @@ -0,0 +1,164 @@ +import copy +from pathlib import Path +from pprint import pprint + +import yaml +import numpy as np + + +def load_yaml_file(file_path: Path): + with open(file_path) as file: + return yaml.full_load(file) + +def generate_client(id, template: dict, world_size: int, type='default', cpu_set=None, num_cpus=1): + local_template = copy.deepcopy(template) + key_name = list(local_template.keys())[0] + container_name = f'client_{type}_{id}' + local_template[container_name] = local_template.pop(key_name) + for key, item in enumerate(local_template[container_name]['environment']): + if item == 'RANK={rank}': + local_template[container_name]['environment'][key] = item.format(rank=id) + if item == 'WORLD_SIZE={world_size}': + local_template[container_name]['environment'][key] = item.format(world_size=world_size) + # for key, item in enumerate(local_template[container_name]): + # if item == 'cpuset: {cpu_set}': + # local_template[container_name][key] = item.format(cpu_set=cpu_set) + + local_template[container_name]['ports'] = [f'{5000+id}:5000'] + if cpu_set: + local_template[container_name]['cpuset'] = f'{cpu_set}' + else: + local_template[container_name].pop('cpuset') + local_template[container_name]['deploy']['resources']['limits']['cpus'] = f'{num_cpus}' + return local_template, container_name + +def gen_client(name: str, client_dict: dict, base_path: Path): + """ + rank (id) + num_cpu + cpu_set + name + """ + client_descr_template = { + 'rank': 0, + 'num_cpu': 1, + 'num_cores': None, + 'name': name, + 'stub-file': 'stub.yml' + } + print(Path.cwd()) + mu = client_dict['cpu-speed'] + sigma = client_dict['cpu-variation'] + n = client_dict['amount'] + np.random.seed(0) + stub_file = base_path / client_dict['stub-name'] + stub_data = load_yaml_file(stub_file) + if client_dict['pin-cores'] is True: + client_descr_template['num_cores'] = client_dict['num-cores'] + client_descr_template['stub-file'] = client_dict['stub-name'] + # print(name) + # pprint(stub_data) + client_cpu_speeds = np.abs(np.round(np.random.normal(mu, sigma, size=n), 2)) + client_descriptions = [] + for cpu_speed in client_cpu_speeds: + client_descr = copy.deepcopy(client_descr_template) + client_descr['num_cpu'] = cpu_speed + client_descriptions.append(client_descr) + # client_data = copy.deepcopy(client_dict) + # client_data.pop('cpu-variation') + # print(cpu_speed) + # print(np.random.normal(mu, sigma, size=n)) + # for k, v in client_dict.items(): + # print(k) + return client_descriptions +def generate_clients_proporties(clients_dict: dict, path: Path): + results = [] + for k,v in clients_dict.items(): + results += gen_client(k, v, path) + return results + +def generate_compose_file(path: Path): + """ + Used properties: + - World size + - num clients? + - path to deploy files + - random seed? + """ + # system = { + # + # 'federator': { + # 'stub-name': 'system_stub.yml', + # 'pin-cores': True, + # 'num-cores': 1 + # }, + # 'clients': { + # 'fast': { + # 'stub-name': 'stub_default.yml', + # 'amount': 1, + # 'pin-cores': True, + # 'num-cores': 3, + # 'cpu-speed': 3, + # 'cpu-variation': 0 + # }, + # 'slow': { + # 'stub-name': 'stub_default.yml', + # 'amount': 0, + # 'pin-cores': True, + # 'num-cores': 1, + # 'cpu-speed': 1, + # 'cpu-variation': 0 + # } + # } + # } + system_path = path / 'description.yml' + system = load_yaml_file(system_path) + # path = Path('deploy/dev_generate') + + client_descriptions = generate_clients_proporties(system['clients'], path) + last_core_id = 0 + world_size = len(client_descriptions) + 1 + system_template_path = path / 'system_stub.yml' + + system_template: dict = load_yaml_file(system_template_path) + + for key, item in enumerate(system_template['services']['fl_server']['environment']): + if item == 'WORLD_SIZE={world_size}': + system_template['services']['fl_server']['environment'][key] = item.format(world_size=world_size) + if system['federator']['pin-cores']: + cpu_set: str + amount = system['federator']['num-cores'] + if amount > 1: + cpu_set = f'{last_core_id}-{last_core_id+amount-1}' + else: + cpu_set = f'{last_core_id}' + system_template['services']['fl_server']['cpuset'] = cpu_set + last_core_id += amount + else: + system_template['services']['fl_server'].pop('cpuset') + + for idx, client_d in enumerate(client_descriptions): + stub_file = path / client_d['stub-file'] + stub_data = load_yaml_file(stub_file) + cpu_set = None + if client_d['num_cores']: + amount = client_d['num_cores'] + if amount > 1: + cpu_set = f'{last_core_id}-{last_core_id+amount-1}' + else: + cpu_set = f'{last_core_id}' + last_core_id += amount + local_template, container_name = generate_client(idx + 1, stub_data, world_size, client_d['name'], cpu_set, client_d['num_cpu']) + system_template['services'].update(local_template) + print(container_name) + + with open(r'./docker-compose.yml', 'w') as file: + yaml.dump(system_template, file, sort_keys=False) + + + +if __name__ == '__main__': + + path = Path('deploy/dev_generate') + results = generate_compose_file(path) + print('done') \ No newline at end of file diff --git a/fltk/util/generate_experiments.py b/fltk/util/generate_experiments.py new file mode 100644 index 00000000..d2f1eb4e --- /dev/null +++ b/fltk/util/generate_experiments.py @@ -0,0 +1,47 @@ +from pathlib import Path + + +def generate(base_path: Path): + descr_path = base_path / 'descr.yaml' + + exp_cfg_list = [x for x in base_path.iterdir() if '.cfg' in x.suffixes] + descr_data = '' + with open(descr_path) as descr_f: + descr_data = descr_f.read() + exps_path = base_path / 'exps' + exps_path.mkdir(parents=True, exist_ok=True) + for exp_cfg in exp_cfg_list: + exp_cfg_data = '' + with open(exp_cfg) as exp_f: + exp_cfg_data = exp_f.read() + + exp_data = descr_data + exp_cfg_data + exp_data += f'\nexperiment_prefix: \'{Path(__file__).parent.name}_{exp_cfg.name.split(".")[0]}\'\n' + filename = '.'.join([exp_cfg.name.split('.')[0], exp_cfg.name.split('.')[2]]) + with open(exps_path / filename, mode='w') as f: + f.write(exp_data) + print('Done') + +# if __name__ == '__main__': +# base_path = Path(__file__).parent +# descr_path = base_path / 'descr.yaml' +# +# exp_cfg_list = [x for x in base_path.iterdir() if '.cfg' in x.suffixes] +# descr_data = '' +# with open(descr_path) as descr_f: +# descr_data = descr_f.read() +# exps_path = base_path / 'exps' +# exps_path.mkdir(parents=True, exist_ok=True) +# for exp_cfg in exp_cfg_list: +# exp_cfg_data = '' +# with open(exp_cfg) as exp_f: +# exp_cfg_data = exp_f.read() +# +# exp_data = descr_data + exp_cfg_data +# exp_data += f'\nexperiment_prefix: \'{Path(__file__).parent.name}_{exp_cfg.name.split(".")[0]}\'\n' +# filename = '.'.join([exp_cfg.name.split('.')[0], exp_cfg.name.split('.')[2]]) +# with open(exps_path / filename, mode='w') as f: +# f.write(exp_data) +# print('Done') +# +#