Feature/setup experiments (#12)

* Add simple reporting * Fix formatting * Add synthetic data experiment config * Add classification support * Fix configs * Fix nonfailing check when pytest fails * Revert changes * Add experiments scripts * Fix classification support * Add early stopping to config * Final preprations * Minor config adjustments * Minor fixes * Hotfix network * Fixes * Fixes * Fix classification * refactor logger interface; adjust experiments and tests to new interface; minor fixes, remove obsolate code * Fix new_classes experiment * Delete results/01_test_data.pkl --------- Co-authored-by: DawidPludowski <[email protected]> Co-authored-by: DawidPludowski <[email protected]>
azoz01 · Oct 26, 2023 · 0433275 · 0433275
1 parent 02d6af4
commit 0433275
Show file tree

Hide file tree

Showing 19 changed files with 171 additions and 264 deletions.
diff --git a/config/01_synthetic_data_experiment_config.yaml b/config/01_synthetic_data_experiment_config.yaml
@@ -1,7 +1,7 @@
 name: "synthetic"
 num_epochs: 100000
 learning_rate: 0.001
-weight_decay: 0.0001
+weight_decay: 0
 batch_size: 256
 gradient_clipping: False
 early_stopping: True
@@ -10,7 +10,7 @@ support_size: 5
 query_size: 27
 
 hidden_representation_size: 32
-n_hidden_layers: 3 # ?
+n_hidden_layers: 3
 hidden_size: 32
 dropout_rate: 0.1
 is_classifier: False

diff --git a/config/02_openml_data_experiment_config.yaml b/config/02_openml_data_experiment_config.yaml
@@ -1,7 +1,7 @@
 name: "openml"
 num_epochs: 100000
 learning_rate: 0.001
-weight_decay: 0.0001
+weight_decay: 0.0
 batch_size: 37
 gradient_clipping: False
 early_stopping: True

diff --git a/config/03_openml_clf_data_experiment_config.yaml b/config/03_openml_clf_data_experiment_config.yaml
@@ -1,7 +1,7 @@
 name: "openml_clf"
 num_epochs: 100000
 learning_rate: 0.001
-weight_decay: 0.0001
+weight_decay: 0
 batch_size: 37
 gradient_clipping: False
 early_stopping: True

diff --git a/config/05_new_classes_experiment_config.yaml b/config/05_new_classes_experiment_config.yaml
@@ -1,16 +1,16 @@
 name: "new_classes"
 num_epochs: 100000
 learning_rate: 0.0001
-weight_decay: 0.0001
-batch_size: 8
+weight_decay: 0
+batch_size: 16
 gradient_clipping: False
 early_stopping: True
 
 support_size: 3
 query_size: 29
 
 hidden_representation_size: 16
-n_hidden_layers: 1
+n_hidden_layers: 3
 hidden_size: 16
 dropout_rate: 0.1
 is_classifier: True

diff --git a/config/06_big_data_experiment_config.yaml b/config/06_big_data_experiment_config.yaml
@@ -2,17 +2,17 @@ name: "big_data"
 classification: true
 num_epochs: 100000
 learning_rate: 0.001
-weight_decay: 0.0001
+weight_decay: 0
 batch_size: 16
 gradient_clipping: False
 early_stopping: True
 
 support_size: 3
 query_size: 29
 
-hidden_representation_size: 16
-n_hidden_layers: 1
-hidden_size: 16
+hidden_representation_size: 32
+n_hidden_layers: 3
+hidden_size: 32
 dropout_rate: 0.1
 is_classifier: True
 

diff --git a/experiments/01_synthetic.py b/experiments/01_synthetic.py
@@ -15,8 +15,7 @@
 from pathlib import Path
 
 
-def main(
-):
+def main():
     config_path = Path("config/01_synthetic_data_experiment_config.yaml")
     logger_type = "both"
     use_profiler = "no"
@@ -65,34 +64,18 @@ def main(
         is_classifier=config["is_classifier"],
     )
 
-    if logger_type == "tb":
-        tb_logger = TensorBoardLogger(
-            "results/tensorboard", 
-            name=config["name"],
-            use_profiler=True if use_profiler == "yes" else False
-        )
-        file_logger = None
-    elif logger_type == "flat":
-        tb_logger = None
-        file_logger = FileLogger("results/flat", name=config["name"])
-    elif logger_type == "both":
-        tb_logger = TensorBoardLogger(
-            "results/tensorboard",
-            name=config["name"],
-            use_profiler=True if use_profiler == "yes" else False
-        )
-        file_logger = FileLogger("results/flat", name=config["name"])
-    else:
-        raise ValueError("logger_type must from [tb, flat, both]")
+    results_path = Path("results") / config["name"]
 
     trainer = HeterogenousAttributesNetworkTrainer(
         n_epochs=config["num_epochs"],
         gradient_clipping=config["gradient_clipping"],
         learning_rate=config["learning_rate"],
         weight_decay=config["weight_decay"],
         early_stopping=config["early_stopping"],
-        file_logger=file_logger,
-        tb_logger=tb_logger,
+        file_logger=True,
+        tb_logger=True,
+        model_checkpoints=True,
+        results_path=results_path,
     )
 
     logger.info("Training model")
@@ -105,4 +88,4 @@ def main(
 
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/experiments/02_openml.py b/experiments/02_openml.py
@@ -13,10 +13,10 @@
 from liltab.train.logger import TensorBoardLogger, FileLogger
 from loguru import logger
 from pathlib import Path
+from torch import nn
 
 
-def main(
-):
+def main():
     config_path = Path("config/02_openml_data_experiment_config.yaml")
     logger_type = "both"
     use_profiler = "no"
@@ -57,44 +57,28 @@ def main(
     )
 
     logger.info("Creating model")
-    from torch import nn
+
     model = HeterogenousAttributesNetwork(
         hidden_representation_size=config["hidden_representation_size"],
         n_hidden_layers=config["n_hidden_layers"],
         hidden_size=config["hidden_size"],
         dropout_rate=config["dropout_rate"],
         is_classifier=config["is_classifier"],
-        inner_activation_function=nn.ELU()
+        inner_activation_function=nn.ReLU(),
     )
 
-    if logger_type == "tb":
-        tb_logger = TensorBoardLogger(
-            "results/tensorboard", 
-            name=config["name"],
-            use_profiler=True if use_profiler == "yes" else False
-        )
-        file_logger = None
-    elif logger_type == "flat":
-        tb_logger = None
-        file_logger = FileLogger("results/flat", name=config["name"])
-    elif logger_type == "both":
-        tb_logger = TensorBoardLogger(
-            "results/tensorboard",
-            name=config["name"],
-            use_profiler=True if use_profiler == "yes" else False
-        )
-        file_logger = FileLogger("results/flat", name=config["name"])
-    else:
-        raise ValueError("logger_type must from [tb, flat, both]")
+    results_path = Path("results") / config["name"]
 
     trainer = HeterogenousAttributesNetworkTrainer(
         n_epochs=config["num_epochs"],
         gradient_clipping=config["gradient_clipping"],
         learning_rate=config["learning_rate"],
         weight_decay=config["weight_decay"],
         early_stopping=config["early_stopping"],
-        file_logger=file_logger,
-        tb_logger=tb_logger,
+        file_logger=True,
+        tb_logger=True,
+        model_checkpoints=True,
+        results_path=results_path,
     )
 
     logger.info("Training model")
@@ -107,4 +91,4 @@ def main(
 
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/experiments/03_openml_clf.py b/experiments/03_openml_clf.py
@@ -1,6 +1,9 @@
 import yaml
 import pytorch_lightning as pl
 
+from pytorch_lightning.callbacks import ModelCheckpoint
+from datetime import datetime
+
 from liltab.data.datasets import PandasDataset
 from liltab.data.dataloaders import (
     FewShotDataLoader,
@@ -16,8 +19,7 @@
 from torch import nn
 
 
-def main(
-):
+def main():
     config_path = Path("config/03_openml_clf_data_experiment_config.yaml")
     logger_type = "both"
     use_profiler = "no"
@@ -32,7 +34,7 @@ def main(
     train_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path(
         Path(config["train_data_path"]),
         PandasDataset,
-        {},
+        {"encode_categorical_target": True},
         FewShotDataLoader,
         {"support_size": config["support_size"], "query_size": config["query_size"]},
         ComposedDataLoader,
@@ -41,7 +43,7 @@ def main(
     val_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path(
         Path(config["val_data_path"]),
         PandasDataset,
-        {},
+        {"encode_categorical_target": True},
         FewShotDataLoader,
         {"support_size": config["support_size"], "query_size": config["query_size"]},
         RepeatableOutputComposedDataLoader,
@@ -50,7 +52,7 @@ def main(
     test_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path(
         Path(config["test_data_path"]),
         PandasDataset,
-        {},
+        {"encode_categorical_target": True},
         FewShotDataLoader,
         {"support_size": config["support_size"], "query_size": config["query_size"]},
         RepeatableOutputComposedDataLoader,
@@ -66,34 +68,19 @@ def main(
         is_classifier=config["is_classifier"],
     )
 
-    if logger_type == "tb":
-        tb_logger = TensorBoardLogger(
-            "results/tensorboard", 
-            name=config["name"],
-            use_profiler=True if use_profiler == "yes" else False
-        )
-        file_logger = None
-    elif logger_type == "flat":
-        tb_logger = None
-        file_logger = FileLogger("results/flat", name=config["name"])
-    elif logger_type == "both":
-        tb_logger = TensorBoardLogger(
-            "results/tensorboard",
-            name=config["name"],
-            use_profiler=True if use_profiler == "yes" else False
-        )
-        file_logger = FileLogger("results/flat", name=config["name"])
-    else:
-        raise ValueError("logger_type must from [tb, flat, both]")
+    results_path = Path("results") / config["name"]
 
     trainer = HeterogenousAttributesNetworkTrainer(
         n_epochs=config["num_epochs"],
         gradient_clipping=config["gradient_clipping"],
         learning_rate=config["learning_rate"],
         weight_decay=config["weight_decay"],
         early_stopping=config["early_stopping"],
-        file_logger=file_logger,
-        tb_logger=tb_logger,
+        loss=nn.CrossEntropyLoss(),
+        file_logger=True,
+        tb_logger=True,
+        model_checkpoints=True,
+        results_path=results_path,
     )
 
     logger.info("Training model")
@@ -102,9 +89,8 @@ def main(
         train_loader=train_loader,
         val_loader=val_loader,
         test_loader=test_loader,
-        loss=nn.CrossEntropyLoss(),
     )
 
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/experiments/04_same_domain.py b/experiments/04_same_domain.py
@@ -15,8 +15,7 @@
 from pathlib import Path
 
 
-def main(
-):
+def main():
     config_path = Path("config/04_same_domain_experiment_config.yaml")
     logger_type = "both"
     use_profiler = "no"
@@ -65,34 +64,18 @@ def main(
         is_classifier=config["is_classifier"],
     )
 
-    if logger_type == "tb":
-        tb_logger = TensorBoardLogger(
-            "results/tensorboard", 
-            name=config["name"],
-            use_profiler=True if use_profiler == "yes" else False
-        )
-        file_logger = None
-    elif logger_type == "flat":
-        tb_logger = None
-        file_logger = FileLogger("results/flat", name=config["name"])
-    elif logger_type == "both":
-        tb_logger = TensorBoardLogger(
-            "results/tensorboard",
-            name=config["name"],
-            use_profiler=True if use_profiler == "yes" else False
-        )
-        file_logger = FileLogger("results/flat", name=config["name"])
-    else:
-        raise ValueError("logger_type must from [tb, flat, both]")
+    results_path = Path("results") / config["name"]
 
     trainer = HeterogenousAttributesNetworkTrainer(
         n_epochs=config["num_epochs"],
         gradient_clipping=config["gradient_clipping"],
         learning_rate=config["learning_rate"],
         weight_decay=config["weight_decay"],
         early_stopping=config["early_stopping"],
-        file_logger=file_logger,
-        tb_logger=tb_logger,
+        file_logger=True,
+        tb_logger=True,
+        model_checkpoints=True,
+        results_path=results_path,
     )
 
     logger.info("Training model")
@@ -105,4 +88,4 @@ def main(
 
 
 if __name__ == "__main__":
-    main()
+    main()