Update configuration object examples for new definition

JMGaljaard · Sep 4, 2022 · 5ab8071 · 5ab8071
1 parent 6c07694
commit 5ab8071
Show file tree

Hide file tree

Showing 15 changed files with 77 additions and 1,118 deletions.
diff --git a/configs/distributed_tasks/example_arrival_config.json b/configs/distributed_tasks/example_arrival_config.json
@@ -1,58 +1,75 @@
-[
-  {
-    "type": "distributed",
-    "jobClassParameters": {
-      "networkConfiguration": {
-        "network": "FashionMNISTCNN",
-        "lossFunction": "CrossEntropyLoss",
-        "dataset": "mnist"
-      },
-      "systemParameters": {
-        "dataParallelism": 2,
-        "configurations": {
-          "default": {
-            "cores": "1000m",
-            "memory": "1Gi"
-          }
-        }
-      },
-      "hyperParameters": {
-        "default": {
-          "batchSize": 128,
-          "testBatchSize": 128,
-          "learningRateDecay": 0.0002,
-          "optimizerConfig": {
-            "type": "Adam",
-            "learningRate": 0.001,
-            "betas": [
-              0.9,
-              0.999
-            ]
+{
+  "experimentConfiguration": {
+    "randomSeed": [
+      1,
+      41,
+      42,
+      43,
+      430
+    ]
+  },
+  "trainTasks": [
+    {
+      "type": "distributed",
+      "lambda": 0.004,
+      "preemptJobs": false,
+      "jobClassParameters": [
+        {
+          "classProbability": 0.1,
+          "priorities": [
+            {
+              "priority": 1,
+              "probability": 0.9
+            },
+            {
+              "priority": 0,
+              "probability": 0.1
+            }
+          ],
+          "networkConfiguration": {
+            "network": "FashionMNISTCNN",
+            "lossFunction": "CrossEntropyLoss",
+            "dataset": "mnist"
+          },
+          "systemParameters": {
+            "dataParallelism": 2,
+            "configurations": {
+              "default": {
+                "cores": "1000m",
+                "memory": "1Gi"
+              }
+            }
+          },
+          "hyperParameters": {
+            "default": {
+              "totalEpochs": 100,
+              "batchSize": 128,
+              "testBatchSize": 128,
+              "learningRateDecay": 0.0002,
+              "optimizerConfig": {
+                "type": "Adam",
+                "learningRate": 0.001,
+                "betas": [
+                  0.9,
+                  0.999
+                ]
+              },
+              "schedulerConfig": {
+                "schedulerStepSize": 50,
+                "schedulerGamma": 0.5,
+                "minimumLearningRate": 1e-10
+              }
+            },
+            "configurations": {
+              "Master": null,
+              "Worker": null
+            }
           },
-          "schedulerConfig": {
-            "schedulerStepSize": 50,
-            "schedulerGamma": 0.5,
-            "minimumLearningRate": 1e-10
+          "learningParameters": {
+            "cuda": false
           }
-        },
-        "configurations": {
-          "Master": null,
-          "Worker": null
         }
-      },
-      "learningParameters": {
-        "totalEpochs": 100,
-        "cuda": false
-      },
-      "experimentConfiguration": {
-        "randomSeed": [
-          1,
-          41,
-          42,
-          43,
-          430
-        ]
-      }
+      ]
     }
-  }
-]
+  ]
+}
diff --git a/configs/example_cloud_experiment.json b/configs/example_cloud_experiment.json
@@ -1,15 +1,14 @@
 {
    "cluster": {
       "orchestrator": {
-         "wait_for_clients": true,
-         "service": "fl-server.test.svc.cluster.local",
-         "nic": "eth0"
+         "orchestrator_type": "simulated"
       },
       "client": {
          "prefix": "client",
          "tensorboard_active": false
       },
-      "image": "gcr.io/test-bed-distml/fltk:latest"
+      "image": "gcr.io/test-bed-distml/fltk:latest",
+      "namespace": "test"
    },
    "execution_config": {
       "duration": 3600,

diff --git a/fltk/datasets/distributed/__init__.py → fltk/datasets/federated/__init__.py b/fltk/datasets/distributed/__init__.py → fltk/datasets/federated/__init__.py
diff --git a/fltk/datasets/distributed/cifar10.py → fltk/datasets/federated/cifar10.py b/fltk/datasets/distributed/cifar10.py → fltk/datasets/federated/cifar10.py
@@ -2,7 +2,7 @@
 from torchvision import datasets
 from torchvision import transforms
 
-from fltk.datasets.distributed.dataset import DistDataset
+from fltk.datasets.distributed import DistDataset
 from fltk.samplers import get_sampler
 from fltk.util.config import FedLearningConfig
 

diff --git a/fltk/datasets/distributed/cifar100.py → fltk/datasets/federated/cifar100.py b/fltk/datasets/distributed/cifar100.py → fltk/datasets/federated/cifar100.py
diff --git a/fltk/datasets/distributed/dataset.py → fltk/datasets/federated/dataset.py b/fltk/datasets/distributed/dataset.py → fltk/datasets/federated/dataset.py
@@ -1,7 +1,5 @@
 from abc import abstractmethod
-from typing import Any
 
-# from fltk.util.arguments import Arguments
 from fltk.util.config import FedLearningConfig
 from fltk.util.log import getLogger
 

diff --git a/fltk/datasets/distributed/fashion_mnist.py → fltk/datasets/federated/fashion_mnist.py b/fltk/datasets/distributed/fashion_mnist.py → fltk/datasets/federated/fashion_mnist.py
diff --git a/fltk/datasets/distributed/mnist.py → fltk/datasets/federated/mnist.py b/fltk/datasets/distributed/mnist.py → fltk/datasets/federated/mnist.py
diff --git a/fltk/datasets/loader_util.py b/fltk/datasets/loader_util.py
diff --git a/fltk/util/generate_data_distribution.py b/fltk/util/generate_data_distribution.py