diff --git a/config/01_synthetic_data_experiment_config.yaml b/config/01_synthetic_data_experiment_config.yaml index bc74bb7..aab4b13 100644 --- a/config/01_synthetic_data_experiment_config.yaml +++ b/config/01_synthetic_data_experiment_config.yaml @@ -1,7 +1,7 @@ name: "synthetic" num_epochs: 100000 learning_rate: 0.001 -weight_decay: 0.0001 +weight_decay: 0 batch_size: 256 gradient_clipping: False early_stopping: True @@ -10,7 +10,7 @@ support_size: 5 query_size: 27 hidden_representation_size: 32 -n_hidden_layers: 3 # ? +n_hidden_layers: 3 hidden_size: 32 dropout_rate: 0.1 is_classifier: False diff --git a/config/02_openml_data_experiment_config.yaml b/config/02_openml_data_experiment_config.yaml index 2ba67a6..e73aa3b 100644 --- a/config/02_openml_data_experiment_config.yaml +++ b/config/02_openml_data_experiment_config.yaml @@ -1,7 +1,7 @@ name: "openml" num_epochs: 100000 learning_rate: 0.001 -weight_decay: 0.0001 +weight_decay: 0.0 batch_size: 37 gradient_clipping: False early_stopping: True diff --git a/config/03_openml_clf_data_experiment_config.yaml b/config/03_openml_clf_data_experiment_config.yaml index ef7b46a..5af8f52 100644 --- a/config/03_openml_clf_data_experiment_config.yaml +++ b/config/03_openml_clf_data_experiment_config.yaml @@ -1,7 +1,7 @@ name: "openml_clf" num_epochs: 100000 learning_rate: 0.001 -weight_decay: 0.0001 +weight_decay: 0 batch_size: 37 gradient_clipping: False early_stopping: True diff --git a/config/05_new_classes_experiment_config.yaml b/config/05_new_classes_experiment_config.yaml index 727d811..d965f81 100644 --- a/config/05_new_classes_experiment_config.yaml +++ b/config/05_new_classes_experiment_config.yaml @@ -1,8 +1,8 @@ name: "new_classes" num_epochs: 100000 learning_rate: 0.0001 -weight_decay: 0.0001 -batch_size: 8 +weight_decay: 0 +batch_size: 16 gradient_clipping: False early_stopping: True @@ -10,7 +10,7 @@ support_size: 3 query_size: 29 hidden_representation_size: 16 -n_hidden_layers: 1 +n_hidden_layers: 3 hidden_size: 16 dropout_rate: 0.1 is_classifier: True diff --git a/config/06_big_data_experiment_config.yaml b/config/06_big_data_experiment_config.yaml index dba42ab..c95d5a4 100644 --- a/config/06_big_data_experiment_config.yaml +++ b/config/06_big_data_experiment_config.yaml @@ -2,7 +2,7 @@ name: "big_data" classification: true num_epochs: 100000 learning_rate: 0.001 -weight_decay: 0.0001 +weight_decay: 0 batch_size: 16 gradient_clipping: False early_stopping: True @@ -10,9 +10,9 @@ early_stopping: True support_size: 3 query_size: 29 -hidden_representation_size: 16 -n_hidden_layers: 1 -hidden_size: 16 +hidden_representation_size: 32 +n_hidden_layers: 3 +hidden_size: 32 dropout_rate: 0.1 is_classifier: True diff --git a/experiments/01_synthetic.py b/experiments/01_synthetic.py index c156ad9..b196d2f 100644 --- a/experiments/01_synthetic.py +++ b/experiments/01_synthetic.py @@ -15,8 +15,7 @@ from pathlib import Path -def main( -): +def main(): config_path = Path("config/01_synthetic_data_experiment_config.yaml") logger_type = "both" use_profiler = "no" @@ -65,25 +64,7 @@ def main( is_classifier=config["is_classifier"], ) - if logger_type == "tb": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = None - elif logger_type == "flat": - tb_logger = None - file_logger = FileLogger("results/flat", name=config["name"]) - elif logger_type == "both": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = FileLogger("results/flat", name=config["name"]) - else: - raise ValueError("logger_type must from [tb, flat, both]") + results_path = Path("results") / config["name"] trainer = HeterogenousAttributesNetworkTrainer( n_epochs=config["num_epochs"], @@ -91,8 +72,10 @@ def main( learning_rate=config["learning_rate"], weight_decay=config["weight_decay"], early_stopping=config["early_stopping"], - file_logger=file_logger, - tb_logger=tb_logger, + file_logger=True, + tb_logger=True, + model_checkpoints=True, + results_path=results_path, ) logger.info("Training model") @@ -105,4 +88,4 @@ def main( if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/experiments/02_openml.py b/experiments/02_openml.py index b5e9016..ea7c99c 100644 --- a/experiments/02_openml.py +++ b/experiments/02_openml.py @@ -13,10 +13,10 @@ from liltab.train.logger import TensorBoardLogger, FileLogger from loguru import logger from pathlib import Path +from torch import nn -def main( -): +def main(): config_path = Path("config/02_openml_data_experiment_config.yaml") logger_type = "both" use_profiler = "no" @@ -57,35 +57,17 @@ def main( ) logger.info("Creating model") - from torch import nn + model = HeterogenousAttributesNetwork( hidden_representation_size=config["hidden_representation_size"], n_hidden_layers=config["n_hidden_layers"], hidden_size=config["hidden_size"], dropout_rate=config["dropout_rate"], is_classifier=config["is_classifier"], - inner_activation_function=nn.ELU() + inner_activation_function=nn.ReLU(), ) - if logger_type == "tb": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = None - elif logger_type == "flat": - tb_logger = None - file_logger = FileLogger("results/flat", name=config["name"]) - elif logger_type == "both": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = FileLogger("results/flat", name=config["name"]) - else: - raise ValueError("logger_type must from [tb, flat, both]") + results_path = Path("results") / config["name"] trainer = HeterogenousAttributesNetworkTrainer( n_epochs=config["num_epochs"], @@ -93,8 +75,10 @@ def main( learning_rate=config["learning_rate"], weight_decay=config["weight_decay"], early_stopping=config["early_stopping"], - file_logger=file_logger, - tb_logger=tb_logger, + file_logger=True, + tb_logger=True, + model_checkpoints=True, + results_path=results_path, ) logger.info("Training model") @@ -107,4 +91,4 @@ def main( if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/experiments/03_openml_clf.py b/experiments/03_openml_clf.py index c73f081..8ff9fa0 100644 --- a/experiments/03_openml_clf.py +++ b/experiments/03_openml_clf.py @@ -1,6 +1,9 @@ import yaml import pytorch_lightning as pl +from pytorch_lightning.callbacks import ModelCheckpoint +from datetime import datetime + from liltab.data.datasets import PandasDataset from liltab.data.dataloaders import ( FewShotDataLoader, @@ -16,8 +19,7 @@ from torch import nn -def main( -): +def main(): config_path = Path("config/03_openml_clf_data_experiment_config.yaml") logger_type = "both" use_profiler = "no" @@ -32,7 +34,7 @@ def main( train_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path( Path(config["train_data_path"]), PandasDataset, - {}, + {"encode_categorical_target": True}, FewShotDataLoader, {"support_size": config["support_size"], "query_size": config["query_size"]}, ComposedDataLoader, @@ -41,7 +43,7 @@ def main( val_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path( Path(config["val_data_path"]), PandasDataset, - {}, + {"encode_categorical_target": True}, FewShotDataLoader, {"support_size": config["support_size"], "query_size": config["query_size"]}, RepeatableOutputComposedDataLoader, @@ -50,7 +52,7 @@ def main( test_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path( Path(config["test_data_path"]), PandasDataset, - {}, + {"encode_categorical_target": True}, FewShotDataLoader, {"support_size": config["support_size"], "query_size": config["query_size"]}, RepeatableOutputComposedDataLoader, @@ -66,25 +68,7 @@ def main( is_classifier=config["is_classifier"], ) - if logger_type == "tb": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = None - elif logger_type == "flat": - tb_logger = None - file_logger = FileLogger("results/flat", name=config["name"]) - elif logger_type == "both": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = FileLogger("results/flat", name=config["name"]) - else: - raise ValueError("logger_type must from [tb, flat, both]") + results_path = Path("results") / config["name"] trainer = HeterogenousAttributesNetworkTrainer( n_epochs=config["num_epochs"], @@ -92,8 +76,11 @@ def main( learning_rate=config["learning_rate"], weight_decay=config["weight_decay"], early_stopping=config["early_stopping"], - file_logger=file_logger, - tb_logger=tb_logger, + loss=nn.CrossEntropyLoss(), + file_logger=True, + tb_logger=True, + model_checkpoints=True, + results_path=results_path, ) logger.info("Training model") @@ -102,9 +89,8 @@ def main( train_loader=train_loader, val_loader=val_loader, test_loader=test_loader, - loss=nn.CrossEntropyLoss(), ) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/experiments/04_same_domain.py b/experiments/04_same_domain.py index c8eb85b..27b5345 100644 --- a/experiments/04_same_domain.py +++ b/experiments/04_same_domain.py @@ -15,8 +15,7 @@ from pathlib import Path -def main( -): +def main(): config_path = Path("config/04_same_domain_experiment_config.yaml") logger_type = "both" use_profiler = "no" @@ -65,25 +64,7 @@ def main( is_classifier=config["is_classifier"], ) - if logger_type == "tb": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = None - elif logger_type == "flat": - tb_logger = None - file_logger = FileLogger("results/flat", name=config["name"]) - elif logger_type == "both": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = FileLogger("results/flat", name=config["name"]) - else: - raise ValueError("logger_type must from [tb, flat, both]") + results_path = Path("results") / config["name"] trainer = HeterogenousAttributesNetworkTrainer( n_epochs=config["num_epochs"], @@ -91,8 +72,10 @@ def main( learning_rate=config["learning_rate"], weight_decay=config["weight_decay"], early_stopping=config["early_stopping"], - file_logger=file_logger, - tb_logger=tb_logger, + file_logger=True, + tb_logger=True, + model_checkpoints=True, + results_path=results_path, ) logger.info("Training model") @@ -105,4 +88,4 @@ def main( if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/experiments/05_new_classes.py b/experiments/05_new_classes.py index a2b8b11..99aed68 100644 --- a/experiments/05_new_classes.py +++ b/experiments/05_new_classes.py @@ -16,8 +16,7 @@ from torch import nn -def main( -): +def main(): config_path = Path("config/05_new_classes_experiment_config.yaml") logger_type = "both" use_profiler = "no" @@ -32,7 +31,7 @@ def main( train_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path( Path(config["train_data_path"]), PandasDataset, - {}, + {"encode_categorical_target": True}, FewShotDataLoader, {"support_size": config["support_size"], "query_size": config["query_size"]}, ComposedDataLoader, @@ -41,7 +40,7 @@ def main( val_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path( Path(config["val_data_path"]), PandasDataset, - {}, + {"encode_categorical_target": True}, FewShotDataLoader, {"support_size": config["support_size"], "query_size": config["query_size"]}, RepeatableOutputComposedDataLoader, @@ -50,7 +49,7 @@ def main( test_loader = ComposedDataLoaderFactory.create_composed_dataloader_from_path( Path(config["test_data_path"]), PandasDataset, - {}, + {"encode_categorical_target": True}, FewShotDataLoader, {"support_size": config["support_size"], "query_size": config["query_size"]}, RepeatableOutputComposedDataLoader, @@ -66,25 +65,7 @@ def main( is_classifier=config["is_classifier"], ) - if logger_type == "tb": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = None - elif logger_type == "flat": - tb_logger = None - file_logger = FileLogger("results/flat", name=config["name"]) - elif logger_type == "both": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = FileLogger("results/flat", name=config["name"]) - else: - raise ValueError("logger_type must from [tb, flat, both]") + results_path = Path("results") / config["name"] trainer = HeterogenousAttributesNetworkTrainer( n_epochs=config["num_epochs"], @@ -92,8 +73,11 @@ def main( learning_rate=config["learning_rate"], weight_decay=config["weight_decay"], early_stopping=config["early_stopping"], - file_logger=file_logger, - tb_logger=tb_logger, + loss=nn.CrossEntropyLoss(), + file_logger=True, + tb_logger=True, + model_checkpoints=True, + results_path=results_path, ) logger.info("Training model") @@ -102,10 +86,8 @@ def main( train_loader=train_loader, val_loader=val_loader, test_loader=test_loader, - loss=nn.CrossEntropyLoss(), - ) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/experiments/06_big_data.py b/experiments/06_big_data.py index c588a8c..1364bb0 100644 --- a/experiments/06_big_data.py +++ b/experiments/06_big_data.py @@ -16,8 +16,7 @@ from torch import nn -def main( -): +def main(): config_path = Path("config/06_big_data_experiment_config.yaml") logger_type = "both" use_profiler = "no" @@ -66,25 +65,7 @@ def main( is_classifier=config["is_classifier"], ) - if logger_type == "tb": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = None - elif logger_type == "flat": - tb_logger = None - file_logger = FileLogger("results/flat", name=config["name"]) - elif logger_type == "both": - tb_logger = TensorBoardLogger( - "results/tensorboard", - name=config["name"], - use_profiler=True if use_profiler == "yes" else False - ) - file_logger = FileLogger("results/flat", name=config["name"]) - else: - raise ValueError("logger_type must from [tb, flat, both]") + results_path = Path("results") / config["name"] trainer = HeterogenousAttributesNetworkTrainer( n_epochs=config["num_epochs"], @@ -92,8 +73,11 @@ def main( learning_rate=config["learning_rate"], weight_decay=config["weight_decay"], early_stopping=config["early_stopping"], - file_logger=file_logger, - tb_logger=tb_logger, + loss=nn.CrossEntropyLoss(), + file_logger=True, + tb_logger=True, + model_checkpoints=True, + results_path=results_path, ) logger.info("Training model") @@ -102,9 +86,8 @@ def main( train_loader=train_loader, val_loader=val_loader, test_loader=test_loader, - loss=nn.CrossEntropyLoss(), ) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/liltab/model/heterogenous_attributes_network.py b/liltab/model/heterogenous_attributes_network.py index f03eda7..2c96a31 100644 --- a/liltab/model/heterogenous_attributes_network.py +++ b/liltab/model/heterogenous_attributes_network.py @@ -44,8 +44,8 @@ def __init__( output_activation_function (Callable, optional): Output activation function of networks used during inference. Should be function from torch.nn. Defaults to nn.Identity(). - is_classifier (bool, optional): If true then output of the network will be - passed through softmax function. Defaults to False. + is_classifier (bool, optional): If true then output of the network will + generate probabilities of classes for query set Defaults to False. """ super().__init__() self.initial_features_encoding_network = FeedForwardNetwork( @@ -55,7 +55,7 @@ def __init__( hidden_size, dropout_rate, inner_activation_function, - output_activation_function, + inner_activation_function, ) self.initial_features_representation_network = FeedForwardNetwork( hidden_representation_size, @@ -64,7 +64,7 @@ def __init__( hidden_size, dropout_rate, inner_activation_function, - output_activation_function, + inner_activation_function, ) self.interaction_encoding_network = FeedForwardNetwork( hidden_representation_size + 1, @@ -73,7 +73,7 @@ def __init__( hidden_size, dropout_rate, inner_activation_function, - output_activation_function, + inner_activation_function, ) self.interaction_representation_network = FeedForwardNetwork( hidden_representation_size, @@ -82,7 +82,7 @@ def __init__( hidden_size, dropout_rate, inner_activation_function, - output_activation_function, + inner_activation_function, ) self.features_encoding_network = FeedForwardNetwork( @@ -92,7 +92,7 @@ def __init__( hidden_size, dropout_rate, inner_activation_function, - output_activation_function, + inner_activation_function, ) self.features_representation_network = FeedForwardNetwork( hidden_representation_size, @@ -101,7 +101,7 @@ def __init__( hidden_size, dropout_rate, inner_activation_function, - output_activation_function, + inner_activation_function, ) self.inference_encoding_network = FeedForwardNetwork( @@ -111,7 +111,7 @@ def __init__( hidden_size, dropout_rate, inner_activation_function, - output_activation_function, + inner_activation_function, ) self.inference_embedding_network = FeedForwardNetwork( hidden_representation_size, @@ -120,7 +120,7 @@ def __init__( hidden_size, dropout_rate, inner_activation_function, - output_activation_function, + inner_activation_function, ) self.inference_network = FeedForwardNetwork( 2 * hidden_representation_size, @@ -539,7 +539,7 @@ def _make_prediction_clf( X_support_inference_embedding, y_support ) response = F.softmax( - torch.cdist(X_query_inference_embedding, classes_representations) ** 2, dim=1 + -(torch.cdist(X_query_inference_embedding, classes_representations) ** 2), dim=1 ) return response @@ -559,7 +559,8 @@ def _calculate_classes_representations(self, X: Tensor, y: Tensor) -> Tensor: y = y.argmax(axis=1) classes_representations = torch.zeros((response_values.shape[0], X.shape[1])) for val in response_values: - classes_representations[val] = X[y == val].mean(axis=0) + if (y == val).sum() != 0: + classes_representations[val] = X[y == val].mean(axis=0) return classes_representations def _get_inference_embedding_of_set( diff --git a/liltab/train/logger.py b/liltab/train/logger.py index e97081a..8115430 100644 --- a/liltab/train/logger.py +++ b/liltab/train/logger.py @@ -1,16 +1,10 @@ -import torch - -from torch import Tensor from pathlib import Path from datetime import datetime -from torch.profiler import profile -from typing import Any, List +from typing import Any from lightning_fabric.utilities.types import _PATH from pytorch_lightning.loggers import TensorBoardLogger as TBLogger -from ..model.heterogenous_attributes_network import HeterogenousAttributesNetwork - class TensorBoardLogger(TBLogger): def __init__( @@ -18,26 +12,13 @@ def __init__( save_dir: _PATH = "results/tensorboard", version: str = None, name: str = "", - use_profiler: bool = False, **kwargs: Any ): if version is None: - _version = "experiment " + name + " " + datetime.now().strftime("%m-%d-%Y-%H:%M:%S") + _version = "experiment_" + name + " " + datetime.now().strftime("%m-%d-%Y-%H:%M:%S") else: _version = version - self.use_profiler = use_profiler - - if self.use_profiler: - self.profiler = profile( - schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=2), - on_trace_ready=torch.profiler.tensorboard_trace_handler( - Path("./results") / "tensorboard" / _version - ), - record_shapes=True, - with_stack=True, - ) - self.iter = 0 super().__init__( @@ -61,26 +42,9 @@ def log_test_value(self, value: float) -> None: def log_validate_value(self, value: float) -> None: self.experiment.add_scalar("validate loss", value, self.iter) - def log_model_graph( - self, model: HeterogenousAttributesNetwork, model_input: List[Tensor] - ) -> None: - self.experiment.add_graph(model, model_input) - def log_weights(self, weights) -> None: self.experiment.add_histogram("weights and biases", weights) - def profile_start(self): - if self.use_profiler: - self.profiler.start() - - def profile_end(self): - if self.use_profiler: - self.profiler.stop() - - def profile_step(self): - if self.use_profiler: - self.profiler.step() - class FileLogger: def __init__(self, save_dir: str = "results/flat", name: str = "", version: str = None) -> None: diff --git a/liltab/train/trainer.py b/liltab/train/trainer.py index 9c3f4ca..6ec8529 100644 --- a/liltab/train/trainer.py +++ b/liltab/train/trainer.py @@ -4,7 +4,7 @@ from pathlib import Path from torch import nn from datetime import datetime -from typing import Union, Callable +from typing import Optional, Callable, Union from liltab.model.heterogenous_attributes_network import HeterogenousAttributesNetwork from liltab.data.dataloaders import ( @@ -27,8 +27,11 @@ def __init__( learning_rate: float, weight_decay: float, early_stopping: bool = False, - file_logger: Union[FileLogger, None] = None, - tb_logger: Union[TensorBoardLogger, None] = None, + loss: Callable = nn.MSELoss(), + file_logger: bool = True, + tb_logger: bool = True, + model_checkpoints: bool = True, + results_path: Union[str, Path, None] = None, ): """ Args: @@ -37,34 +40,80 @@ def __init__( learning_rate (float): learning rate during training. weight_decay (float): weight decay during training. early_stopping (Optional, bool): if True, then early stopping with - patience n_epochs // 5 is applied. Defaults to False. - file_logger (FileLogger|None): csv logger - tb_logger (TensorBoardLogger|None): tensorboard logger + patience n_epochs // 10 is applied. Defaults to False. + loss (Callable): Loss used during training. Defaults to MSELoss(). + file_logger (bool): if True, then file logger will write to + {results_path} directory + tb_logger (bool): if True, then tensorboard logger will write to + {results_path} directory + model_checkpoints (bool): if True, then model checkpoints will + be loaded to {results_path} directory + results_path (Optional, str, Path): directory to save logs and + model checkpoints; required only if any of `file_logger`, + `tb_logger`, `model_checkpoints` is not None """ - callbacks = LoggerCallback(file_logger=file_logger, tb_logger=tb_logger) - model_path = Path("results") - model_path = model_path / "models" / datetime.now().strftime("%m-%d-%Y-%H:%M:%S") - model_checkpoints = ModelCheckpoint( - dirpath=model_path, - filename="model-{epoch}-{val_loss:.2f}", - save_top_k=1, - mode="min", - every_n_epochs=10, - save_last=True, - ) - callbacks = [callbacks, model_checkpoints] + + if not results_path and (file_logger or tb_logger or model_checkpoints): + raise ValueError( + """`results_path` is required if any of (`file_logger`, + `tb_logger`, `model_checkpoints`) is not None""" + ) + + callbacks = [] + + timestamp = datetime.now().strftime("%m-%d-%Y-%H:%M:%S") + + if file_logger: + file_logger_callback = FileLogger( + save_dir=Path(results_path) / timestamp, version="flat" + ) + else: + file_logger_callback = None + + if tb_logger: + tb_logger_callback = TensorBoardLogger( + save_dir=Path(results_path) / timestamp, version="tensorboard" + ) + else: + tb_logger_callback = None + + if file_logger or tb_logger: + loggers_callback = LoggerCallback( + file_logger=file_logger_callback, tb_logger=tb_logger_callback + ) + callbacks.append(loggers_callback) + if early_stopping: - early_stopping = EarlyStopping(monitor="val_loss", mode="min", patience=n_epochs // 5) + early_stopping = EarlyStopping( + monitor="val_loss", + mode="min", + patience=100, + min_delta=1e-3, + ) callbacks.append(early_stopping) + if model_checkpoints: + model_checkpoints_callback = ModelCheckpoint( + dirpath=Path(results_path) / timestamp / "model_checkpoints", + filename="model-{epoch}-{val_loss:.2f}", + save_top_k=1, + mode="min", + every_n_epochs=10, + save_last=True, + ) + callbacks.append(model_checkpoints_callback) + + check_val_every_n_epoch = n_epochs // 1000 if n_epochs > 1000 else 1 + self.trainer = pl.Trainer( max_epochs=n_epochs, gradient_clip_val=1 if gradient_clipping else 0, - check_val_every_n_epoch=n_epochs // 1000 if n_epochs > 1000 else 1, + check_val_every_n_epoch=check_val_every_n_epoch, callbacks=callbacks, ) self.learning_rate = learning_rate self.weight_decay = weight_decay + self.loss = loss def train_and_test( self, @@ -72,7 +121,6 @@ def train_and_test( train_loader: ComposedDataLoader | RepeatableOutputComposedDataLoader, val_loader: ComposedDataLoader | RepeatableOutputComposedDataLoader, test_loader: ComposedDataLoader | RepeatableOutputComposedDataLoader, - loss: Callable = nn.MSELoss(), ) -> tuple[LightningWrapper, list[dict[str, float]]]: """ Method used to train and test model. @@ -85,14 +133,16 @@ def train_and_test( loader with validation data test_loader (ComposedDataLoader | RepeatableOutputComposedDataLoader): loader with test data - loss (Callable): Loss used during training. Defaults to MSELoss(). Returns: tuple[HeterogenousAttributesNetwork, list[dict[str, float]]]: trained network with metrics on test set. """ model_wrapper = LightningWrapper( - model, learning_rate=self.learning_rate, weight_decay=self.weight_decay, loss=loss + model, + learning_rate=self.learning_rate, + weight_decay=self.weight_decay, + loss=self.loss, ) self.trainer.fit(model_wrapper, train_loader, val_loader) test_results = self.trainer.test(model_wrapper, test_loader) @@ -102,8 +152,8 @@ def train_and_test( class LoggerCallback(Callback): def __init__( self, - file_logger: Union[FileLogger, None] = None, - tb_logger: Union[TensorBoardLogger, None] = None, + file_logger: Optional[FileLogger] = None, + tb_logger: Optional[TensorBoardLogger] = None, ) -> None: """ Args: @@ -147,16 +197,3 @@ def on_test_batch_end( if self.tb_logger is not None: self.tb_logger.log_test_value(loss_value) - - def on_fit_end(self, trainer, pl_module): - if self.tb_logger is not None: - self.tb_logger.log_model_graph(pl_module.model, pl_module.example_input) - self.tb_logger.profile_end() - - def on_train_epoch_end(self, trainer, pl_module) -> None: - if self.tb_logger is not None: - self.tb_logger.profile_step() - - def on_fit_start(self, trainer, pl_module) -> None: - if self.tb_logger is not None: - self.tb_logger.profile_start() diff --git a/liltab/train/utils.py b/liltab/train/utils.py index 34c5d06..87f664c 100644 --- a/liltab/train/utils.py +++ b/liltab/train/utils.py @@ -35,13 +35,9 @@ def __init__( self.weight_decay = weight_decay self.metrics_history = dict() - self.example_input = None self.save_hyperparameters() def training_step(self, batch: list[tuple[Tensor, Tensor, Tensor, Tensor]], batch_idx) -> float: - if batch_idx == 0: - self.example_input = batch[0][:3] - sum_loss_value = 0.0 for i, example in enumerate(batch): X_support, y_support, X_query, y_query = example @@ -84,3 +80,6 @@ def test_step(self, batch: list[tuple[Tensor, Tensor, Tensor, Tensor]], batch_id def configure_optimizers(self) -> Any: return optim.Adam(self.parameters(), lr=self.learning_rate, weight_decay=self.weight_decay) + + def calculate_model_weights_norm(self) -> Tensor: + pass diff --git a/results/.gitignore b/results/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/results/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/test/liltab/data/test_dataloaders.py b/test/liltab/data/test_dataloaders.py index c635fd3..d4aed66 100644 --- a/test/liltab/data/test_dataloaders.py +++ b/test/liltab/data/test_dataloaders.py @@ -1,6 +1,3 @@ -# ComposedDataLoader (dwa loadery) -# Czy hasnext dziala dobrze - import numpy as np from liltab.data.datasets import PandasDataset diff --git a/test/liltab/model/test_heterogenous_attributes_network.py b/test/liltab/model/test_heterogenous_attributes_network.py index c96346d..75ffb92 100644 --- a/test/liltab/model/test_heterogenous_attributes_network.py +++ b/test/liltab/model/test_heterogenous_attributes_network.py @@ -328,7 +328,7 @@ def test_get_inference_embedding_of_set(): expected_query_example_embedding = network._enrich_representation_with_set_rows( attributes_representation, expected_query_example_embedding ) - expected_query_example_embedding = expected_query_example_embedding.reshape(-1, 33) + expected_query_example_embedding = expected_query_example_embedding.reshape(10, 33) expected_query_example_embedding = network.inference_encoding_network( expected_query_example_embedding ).mean(axis=0) diff --git a/test/liltab/train/test_trainer.py b/test/liltab/train/test_trainer.py index 5a84517..77cfec5 100644 --- a/test/liltab/train/test_trainer.py +++ b/test/liltab/train/test_trainer.py @@ -31,6 +31,12 @@ def test_lighting_wrapper(resources_path): model = HeterogenousAttributesNetwork() trainer = HeterogenousAttributesNetworkTrainer( - n_epochs=1, gradient_clipping=True, learning_rate=1e-3, weight_decay=0.1 + n_epochs=1, + gradient_clipping=True, + learning_rate=1e-3, + weight_decay=0.1, + file_logger=False, + tb_logger=False, + model_checkpoints=False, ) trainer.train_and_test(model, dataloader_1, dataloader_2, dataloader_3)