From 34681ae676cac61aa88731a33652717e4a2a5a1b Mon Sep 17 00:00:00 2001 From: Gustavo Rosa Date: Sun, 8 Jan 2023 15:53:52 -0300 Subject: [PATCH] fix(dbn): Fixes adding missing DBN layers when they are not supplied. --- docs/conf.py | 4 +- .../bernoulli/conv_rbm_classification.py | 10 +- .../deep/conv_dbn_classification.py | 32 +++-- .../gaussian_conv_rbm_classification.py | 10 +- learnergy/__init__.py | 2 +- learnergy/core/dataset.py | 13 +- learnergy/math/scale.py | 2 +- learnergy/models/bernoulli/conv_rbm.py | 25 ++-- .../models/bernoulli/discriminative_rbm.py | 14 +- learnergy/models/bernoulli/dropout_rbm.py | 6 +- learnergy/models/bernoulli/e_dropout_rbm.py | 8 +- learnergy/models/bernoulli/rbm.py | 18 +-- learnergy/models/deep/conv_dbn.py | 127 ++++++++++-------- learnergy/models/deep/dbn.py | 97 ++++++------- learnergy/models/deep/residual_dbn.py | 12 +- learnergy/models/extra/__init__.py | 2 +- learnergy/models/extra/sigmoid_rbm.py | 10 +- learnergy/models/gaussian/__init__.py | 5 +- .../models/gaussian/gaussian_conv_rbm.py | 37 ++--- learnergy/models/gaussian/gaussian_rbm.py | 28 ++-- learnergy/utils/logging.py | 6 +- requirements.txt | 3 - setup.py | 6 +- 23 files changed, 244 insertions(+), 233 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 04d6109..0083c71 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,10 +27,10 @@ author = "Mateus Roder and Gustavo de Rosa" # The short X.Y version -version = "1.1.3" +version = "1.1.4" # The full version, including alpha/beta/rc tags -release = "1.1.3" +release = "1.1.4" # -- General configuration --------------------------------------------------- diff --git a/examples/applications/bernoulli/conv_rbm_classification.py b/examples/applications/bernoulli/conv_rbm_classification.py index 7ca85e2..14805dd 100644 --- a/examples/applications/bernoulli/conv_rbm_classification.py +++ b/examples/applications/bernoulli/conv_rbm_classification.py @@ -52,10 +52,10 @@ nf = model.n_filters if model.maxpooling: - input_fc = nf * (h1//2 + 1) * (h2//2 + 1) + input_fc = nf * (h1 // 2 + 1) * (h2 // 2 + 1) else: input_fc = nf * h1 * h2 -fc = nn.Linear(input_fc , n_classes) +fc = nn.Linear(input_fc, n_classes) # Check if model uses GPU if model.device == "cuda": @@ -99,8 +99,7 @@ y = model(x_batch) # Reshaping the outputs - y = y.reshape( - x_batch.size(0), input_fc) + y = y.reshape(x_batch.size(0), input_fc) # Calculating the fully-connected outputs y = fc(y) @@ -131,8 +130,7 @@ y = model(x_batch) # Reshaping the outputs - y = y.reshape( - x_batch.size(0), input_fc) + y = y.reshape(x_batch.size(0), input_fc) # Calculating the fully-connected outputs y = fc(y) diff --git a/examples/applications/deep/conv_dbn_classification.py b/examples/applications/deep/conv_dbn_classification.py index 50dd4d2..371a854 100644 --- a/examples/applications/deep/conv_dbn_classification.py +++ b/examples/applications/deep/conv_dbn_classification.py @@ -1,10 +1,10 @@ import torch -import torchvision import torch.nn as nn import torch.optim as optim - +import torchvision from torch.utils.data import DataLoader from tqdm import tqdm + from learnergy.models.deep import ConvDBN # Creating training and testing dataset @@ -33,7 +33,7 @@ momentum=(0, 0), decay=(0, 0), maxpooling=(True, False), - #pooling_kernel=(2, 0), # WORKING ON ... + # pooling_kernel=(2, 0), # WORKING ON ... use_gpu=True, ) @@ -46,22 +46,22 @@ model.fit(train, batch_size=batch_size, epochs=epochs) # Reconstructing test set -#rec_mse, v = model.reconstruct(test) +# rec_mse, v = model.reconstruct(test) # Saving model torch.save(model, "model.pth") # Creating the Fully Connected layer to append on top of DBN -h1 = model.models[len(model.models)-1].hidden_shape[0] -h2 = model.models[len(model.models)-1].hidden_shape[1] -nf = model.models[len(model.models)-1].n_filters +h1 = model.models[len(model.models) - 1].hidden_shape[0] +h2 = model.models[len(model.models) - 1].hidden_shape[1] +nf = model.models[len(model.models) - 1].n_filters -if model.models[len(model.models)-1].maxpooling: - input_fc = nf * (h1//2 + 1) * (h2//2 + 1) - print('pooling', input_fc) +if model.models[len(model.models) - 1].maxpooling: + input_fc = nf * (h1 // 2 + 1) * (h2 // 2 + 1) + print("pooling", input_fc) else: input_fc = nf * h1 * h2 -fc = nn.Linear(input_fc , n_classes) +fc = nn.Linear(input_fc, n_classes) # Check if model uses GPU if model.device == "cuda": @@ -101,10 +101,9 @@ # Passing the batch down the model y = model(x_batch) - + # Reshaping the outputs - y = y.reshape( - x_batch.size(0), input_fc) + y = y.reshape(x_batch.size(0), input_fc) # Calculating the fully-connected outputs y = fc(y) @@ -133,10 +132,9 @@ # Passing the batch down the model y = model(x_batch) - + # Reshaping the outputs - y = y.reshape( - x_batch.size(0), input_fc) + y = y.reshape(x_batch.size(0), input_fc) # Calculating the fully-connected outputs y = fc(y) diff --git a/examples/applications/gaussian/gaussian_conv_rbm_classification.py b/examples/applications/gaussian/gaussian_conv_rbm_classification.py index f7c5af9..e2f742b 100644 --- a/examples/applications/gaussian/gaussian_conv_rbm_classification.py +++ b/examples/applications/gaussian/gaussian_conv_rbm_classification.py @@ -52,10 +52,10 @@ nf = model.n_filters if model.maxpooling: - input_fc = nf * (h1//2 + 1) * (h2//2 + 1) + input_fc = nf * (h1 // 2 + 1) * (h2 // 2 + 1) else: input_fc = nf * h1 * h2 -fc = nn.Linear(input_fc , n_classes) +fc = nn.Linear(input_fc, n_classes) # Check if model uses GPU if model.device == "cuda": @@ -99,8 +99,7 @@ y = model(x_batch) # Reshaping the outputs - y = y.reshape( - x_batch.size(0), input_fc) + y = y.reshape(x_batch.size(0), input_fc) # Calculating the fully-connected outputs y = fc(y) @@ -131,8 +130,7 @@ y = model(x_batch) # Reshaping the outputs - y = y.reshape( - x_batch.size(0), input_fc) + y = y.reshape(x_batch.size(0), input_fc) # Calculating the fully-connected outputs y = fc(y) diff --git a/learnergy/__init__.py b/learnergy/__init__.py index 54b31e3..5381636 100644 --- a/learnergy/__init__.py +++ b/learnergy/__init__.py @@ -2,4 +2,4 @@ of several modules and sub-modules. """ -__version__ = "1.1.3" +__version__ = "1.1.4" diff --git a/learnergy/core/dataset.py b/learnergy/core/dataset.py index ce0797c..aa08b75 100644 --- a/learnergy/core/dataset.py +++ b/learnergy/core/dataset.py @@ -2,7 +2,6 @@ """ from typing import Optional, Tuple -from xmlrpc.client import Boolean import numpy as np import torch @@ -17,8 +16,11 @@ class Dataset(torch.utils.data.Dataset): """A custom dataset class, inherited from PyTorch's dataset.""" def __init__( - self, data: np.array, targets: np.array, - transform: Optional[callable] = None, show_log: Optional[Boolean] = True + self, + data: np.array, + targets: np.array, + transform: Optional[callable] = None, + show_log: Optional[bool] = True, ) -> None: """Initialization method. @@ -26,6 +28,7 @@ def __init__( data: An n-dimensional array containing the data. targets: An 1-dimensional array containing the data's labels. transform: Optional transform to be applied over a sample. + show_log: Whether to show log information or not. """ @@ -83,7 +86,7 @@ def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]: idx: The idx of desired sample. Returns: - (Tuple[torch.Tensor, torch.Tensor]): Data and label tensors. + Data and label tensors. """ @@ -99,7 +102,7 @@ def __len__(self) -> int: """A private method that will be the base for PyTorch's iterator getting dataset's length. Returns: - (int): Length of dataset. + Length of dataset. """ diff --git a/learnergy/math/scale.py b/learnergy/math/scale.py index 4981ee5..4fea41f 100644 --- a/learnergy/math/scale.py +++ b/learnergy/math/scale.py @@ -13,7 +13,7 @@ def unitary_scale(x: np.array) -> np.array: x: A numpy array to be scaled. Returns: - (np.array): Scaled array. + Scaled array. """ diff --git a/learnergy/models/bernoulli/conv_rbm.py b/learnergy/models/bernoulli/conv_rbm.py index a65ba9b..8c7c802 100644 --- a/learnergy/models/bernoulli/conv_rbm.py +++ b/learnergy/models/bernoulli/conv_rbm.py @@ -55,7 +55,7 @@ def __init__( momentum: Momentum parameter. decay: Weight decay used for penalization. maxpooling: Whether MaxPooling2D should be used or not. - pooling_kernel: The kernel size of MaxPooling layer (when maxpooling=True). + pooling_kernel: The kernel size of MaxPooling2D layer (when maxpooling=True). use_gpu: Whether GPU should be used or not. """ @@ -80,7 +80,9 @@ def __init__( self.decay = decay if maxpooling: - self.maxpol2d = nn.MaxPool2d(kernel_size=pooling_kernel, stride=2, padding=1) + self.maxpol2d = nn.MaxPool2d( + kernel_size=pooling_kernel, stride=2, padding=1 + ) self.maxpooling = True else: self.maxpol2d = maxpooling @@ -115,7 +117,8 @@ def __init__( self.momentum, self.decay, self.maxpooling, - pooling_kernel, pooling_kernel, + pooling_kernel, + pooling_kernel, ) @property @@ -233,7 +236,7 @@ def decay(self, decay: float) -> None: @property def maxpooling(self) -> bool: - """Usage of MaxPooling.""" + """Usage of MaxPooling2D.""" return self._maxpooling @@ -291,7 +294,7 @@ def hidden_sampling(self, v: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: v: A tensor incoming from the visible layer. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -308,7 +311,7 @@ def visible_sampling(self, h: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor] h: A tensor incoming from the hidden layer. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the visible layer sampling. + The probabilities and states of the visible layer sampling. """ @@ -327,7 +330,7 @@ def gibbs_sampling( v: A tensor incoming from the visible layer. Returns: - (Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling (positive), + The probabilities and states of the hidden layer sampling (positive), the probabilities and states of the hidden layer sampling (negative) and the states of the visible layer sampling (negative). @@ -362,7 +365,7 @@ def energy(self, samples: torch.Tensor) -> torch.Tensor: samples: Samples to be energy-freed. Returns: - (torch.Tensor): The system's energy based on input samples. + The system's energy based on input samples. """ @@ -392,7 +395,7 @@ def fit( epochs: Number of training epochs. Returns: - (float): MSE (mean squared error) from the training step. + MSE (mean squared error) from the training step. """ @@ -454,7 +457,7 @@ def reconstruct( dataset: A Dataset object containing the testing data. Returns: - (Tuple[float, torch.Tensor]): Reconstruction error and visible probabilities, i.e., P(v|h). + Reconstruction error and visible probabilities, i.e., P(v|h). """ @@ -498,7 +501,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: x: An input tensor for computing the forward pass. Returns: - (torch.Tensor): A tensor containing the Convolutional RBM's outputs. + A tensor containing the Convolutional RBM's outputs. """ diff --git a/learnergy/models/bernoulli/discriminative_rbm.py b/learnergy/models/bernoulli/discriminative_rbm.py index da7a851..dd595ba 100644 --- a/learnergy/models/bernoulli/discriminative_rbm.py +++ b/learnergy/models/bernoulli/discriminative_rbm.py @@ -132,7 +132,7 @@ def labels_sampling(self, samples: torch.Tensor) -> torch.Tensor: samples: Samples to be labels-calculated. Returns: - (torch.Tensor): Labels' probabilities based on input samples. + Labels' probabilities based on input samples. """ @@ -164,7 +164,7 @@ def fit( epochs: Number of training epochs. Returns: - (Tuple[float, float]): Loss and accuracy from the training step. + Loss and accuracy from the training step. """ @@ -223,7 +223,7 @@ def predict( dataset: A Dataset object containing the testing data. Returns: - (Tuple[float, torch.Tensor, torch.Tensor]): Accuracy, prediction probabilities and labels, i.e., P(y|v). + Accuracy, prediction probabilities and labels, i.e., P(y|v). """ @@ -334,7 +334,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -356,7 +356,7 @@ def class_sampling(self, h: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: h: A tensor incoming from the hidden layer. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the class layer sampling. + The probabilities and states of the class layer sampling. """ @@ -378,7 +378,7 @@ def gibbs_sampling( y: A tensor incoming from the class layer. Returns: - (Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling (positive), + The probabilities and states of the hidden layer sampling (positive), the probabilities and states of the hidden layer sampling (negative) and the states of the visible layer sampling (negative). @@ -420,7 +420,7 @@ def fit( epochs: Number of training epochs. Returns: - (Tuple[float, float]): Loss and accuracy from the training step. + Loss and accuracy from the training step. """ diff --git a/learnergy/models/bernoulli/dropout_rbm.py b/learnergy/models/bernoulli/dropout_rbm.py index 64ee5ff..f346082 100644 --- a/learnergy/models/bernoulli/dropout_rbm.py +++ b/learnergy/models/bernoulli/dropout_rbm.py @@ -93,7 +93,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -126,7 +126,7 @@ def reconstruct( dataset: A Dataset object containing the testing data. Returns: - (Tuple[float, torch.Tensor]): Reconstruction error and visible probabilities, i.e., P(v|h). + Reconstruction error and visible probabilities, i.e., P(v|h). """ @@ -230,7 +230,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ diff --git a/learnergy/models/bernoulli/e_dropout_rbm.py b/learnergy/models/bernoulli/e_dropout_rbm.py index dd21f3e..72dd011 100644 --- a/learnergy/models/bernoulli/e_dropout_rbm.py +++ b/learnergy/models/bernoulli/e_dropout_rbm.py @@ -89,7 +89,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (torch.Tensor): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -112,7 +112,7 @@ def total_energy(self, h: torch.Tensor, v: torch.Tensor) -> torch.Tensor: v: Visible sampling states. Returns: - (torch.Tensor): The total energy of the model. + The total energy of the model. """ @@ -160,7 +160,7 @@ def fit( epochs: Number of training epochs. Returns: - (Tuple[float, float]): MSE (mean squared error) and log pseudo-likelihood from the training step. + MSE (mean squared error) and log pseudo-likelihood from the training step. """ @@ -240,7 +240,7 @@ def reconstruct( dataset: A Dataset object containing the testing data. Returns: - (Tuple[float, torch.Tensor]): Reconstruction error and visible probabilities, i.e., P(v|h). + Reconstruction error and visible probabilities, i.e., P(v|h). """ diff --git a/learnergy/models/bernoulli/rbm.py b/learnergy/models/bernoulli/rbm.py index 654c6cd..89d41a6 100644 --- a/learnergy/models/bernoulli/rbm.py +++ b/learnergy/models/bernoulli/rbm.py @@ -231,7 +231,7 @@ def pre_activation( scale: A boolean to decide whether temperature should be used or not. Returns: - (torch.Tensor): An input for any type of activation function. + An input for any type of activation function. """ @@ -252,7 +252,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (torch.Tensor): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -277,7 +277,7 @@ def visible_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (torch.Tensor): The probabilities and states of the visible layer sampling. + The probabilities and states of the visible layer sampling. """ @@ -301,7 +301,7 @@ def gibbs_sampling( v: A tensor incoming from the visible layer. Returns: - (Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling (positive), + The probabilities and states of the hidden layer sampling (positive), the probabilities and states of the hidden layer sampling (negative) and the states of the visible layer sampling (negative). @@ -332,7 +332,7 @@ def energy(self, samples: torch.Tensor) -> torch.Tensor: samples: Samples to be energy-freed. Returns: - (torch.Tensor): The system's energy based on input samples. + The system's energy based on input samples. """ @@ -355,7 +355,7 @@ def pseudo_likelihood(self, samples: torch.Tensor) -> torch.Tensor: samples: Samples to be calculated. Returns: - (torch.Tensor): The logarithm of the pseudo-likelihood based on input samples. + The logarithm of the pseudo-likelihood based on input samples. """ @@ -395,7 +395,7 @@ def fit( epochs: Number of training epochs. Returns: - (Tuple[float, float]): MSE (mean squared error) and log pseudo-likelihood from the training step. + MSE (mean squared error) and log pseudo-likelihood from the training step. """ @@ -456,7 +456,7 @@ def reconstruct( dataset: A Dataset object containing the testing data. Returns: - (Tuple[float, torch.Tensor]): Reconstruction error and visible probabilities, i.e., P(v|h). + Reconstruction error and visible probabilities, i.e., P(v|h). """ @@ -495,7 +495,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: x: An input tensor for computing the forward pass. Returns: - (torch.Tensor): A tensor containing the RBM's outputs. + A tensor containing the RBM's outputs. """ diff --git a/learnergy/models/deep/conv_dbn.py b/learnergy/models/deep/conv_dbn.py index ac8fc40..7dba211 100644 --- a/learnergy/models/deep/conv_dbn.py +++ b/learnergy/models/deep/conv_dbn.py @@ -11,12 +11,16 @@ import learnergy.utils.exception as e from learnergy.core import Dataset, Model from learnergy.models.bernoulli import ConvRBM -from learnergy.models.gaussian import GaussianConvRBM, GaussianConvRBM4deep +from learnergy.models.gaussian import GaussianConvRBM, GaussianConvRBM4Deep from learnergy.utils import logging logger = logging.get_logger(__name__) -MODELS = {"bernoulli": ConvRBM, "gaussian": GaussianConvRBM, "gaussiandeep": GaussianConvRBM4deep} +MODELS = { + "bernoulli": ConvRBM, + "gaussian": GaussianConvRBM, + "gaussiandeep": GaussianConvRBM4Deep, +} class ConvDBN(Model): @@ -40,8 +44,8 @@ def __init__( learning_rate: Optional[Tuple[float, ...]] = (0.0001,), momentum: Optional[Tuple[float, ...]] = (0.0,), decay: Optional[Tuple[float, ...]] = (0.0,), - maxpooling: Optional[Tuple[bool,...]] = (False, False,), - pooling_kernel: Optional[Tuple[int, ...]] = (2, 2,), + maxpooling: Optional[Tuple[bool, ...]] = (False,), + pooling_kernel: Optional[Tuple[int, ...]] = (2,), use_gpu: Optional[bool] = False, ): """Initialization method. @@ -57,7 +61,7 @@ def __init__( momentum: Momentum parameter per layer. decay: Weight decay used for penalization per layer. maxpooling: Whether MaxPooling2D should be used or not. - pooling_kernel: The kernel size of each square-sized MaxPooling layer (when maxpooling=True). + pooling_kernel: The kernel size of each square-sized MaxPooling2D layer (when maxpooling=True). use_gpu: Whether GPU should be used or not. """ @@ -76,57 +80,61 @@ def __init__( self.steps = steps self.lr = learning_rate self.momentum = momentum - self.decay = decay + self.decay = decay self.maxpooling = maxpooling self.maxpol2d = [] for i, mx in enumerate(maxpooling): if mx: - self.maxpol2d.append(nn.MaxPool2d(kernel_size=pooling_kernel[i], stride=2, padding=1)) + self.maxpol2d.append( + nn.MaxPool2d(kernel_size=pooling_kernel[i], stride=2, padding=1) + ) else: self.maxpol2d.append(None) self.models = [] for i in range(self.n_layers): - - if i > 0 and model=='gaussian': - m = MODELS['gaussiandeep']( - visible_shape, - self.filter_shape[i], - self.n_filters[i], - n_channels, - self.steps[i], - self.lr[i], - self.momentum[i], - self.decay[i], - self.maxpooling[i], - pooling_kernel[i], - use_gpu, - ) + + if i > 0 and model == "gaussian": + m = MODELS["gaussiandeep"]( + visible_shape, + self.filter_shape[i], + self.n_filters[i], + n_channels, + self.steps[i], + self.lr[i], + self.momentum[i], + self.decay[i], + self.maxpooling[i], + pooling_kernel[i], + use_gpu, + ) else: m = MODELS[model]( - visible_shape, - self.filter_shape[i], - self.n_filters[i], - n_channels, - self.steps[i], - self.lr[i], - self.momentum[i], - self.decay[i], - self.maxpooling[i], - pooling_kernel[i], - use_gpu, - ) + visible_shape, + self.filter_shape[i], + self.n_filters[i], + n_channels, + self.steps[i], + self.lr[i], + self.momentum[i], + self.decay[i], + self.maxpooling[i], + pooling_kernel[i], + use_gpu, + ) visible_shape = ( visible_shape[0] - self.filter_shape[i][0] + 1, visible_shape[1] - self.filter_shape[i][1] + 1, ) if self.maxpooling[i]: - # TODO: Needs to be adjusted to when pooling_kernel != 2 - visible_shape = ((m.hidden_shape[0]//2) + 1, (m.hidden_shape[1]//2) + 1) - + # TODO: Needs to be adjusted to when pooling_kernel != 2 + visible_shape = ( + (m.hidden_shape[0] // 2) + 1, + (m.hidden_shape[1] // 2) + 1, + ) n_channels = self.n_filters[i] @@ -247,14 +255,14 @@ def decay(self, decay: Tuple[float, ...]) -> None: @property def maxpooling(self) -> Tuple[bool, ...]: - """Usage of MaxPooling.""" + """Usage of MaxPooling2D.""" return self._maxpooling @maxpooling.setter def maxpooling(self, maxpooling: Tuple[bool, ...]) -> None: if len(maxpooling) != self.n_layers: - raise e.ValueError("`maxpooling` should be a Tuple of True or False") + raise e.SizeError(f"`maxpooling` should have size equal as {self.n_layers}") self._maxpooling = maxpooling @@ -282,7 +290,7 @@ def fit( epochs: Number of training epochs per layer. Returns: - (float): MSE (mean squared error) from the training step. + MSE (mean squared error) from the training step. """ @@ -291,19 +299,19 @@ def fit( mse = [] - try: + try: samples, targets, transform = ( - dataset.data.numpy(), - dataset.targets.numpy(), - dataset.transform, + dataset.data.numpy(), + dataset.targets.numpy(), + dataset.transform, ) d = Dataset(samples, targets, transform) except: try: samples, targets, transform = ( - dataset.data, - dataset.targets, - dataset.transform, + dataset.data, + dataset.targets, + dataset.transform, ) d = Dataset(samples, targets, transform) except: @@ -313,16 +321,17 @@ def fit( for i, model in enumerate(self.models): logger.info("Fitting layer %d/%d ...", i + 1, self.n_layers) - if i ==0: + if i == 0: model_mse = model.fit(d, batch_size, epochs[i]) mse.append(model_mse) - else: - # creating the training phase for deeper models + else: + # Creates the training phase for deeper models for ep in range(epochs[i]): logger.info("Epoch %d/%d", ep + 1, epochs[i]) + model_mse = 0 - for step, (samples, y) in enumerate(batches): + for _, (samples, y) in enumerate(batches): if self.device == "cuda": samples = samples.cuda() @@ -331,14 +340,16 @@ def fit( if self.maxpooling[ii]: samples = self.maxpol2d[ii](samples) - # Creating the dataset to ''mini-fit'' the i-th model + # Creating the dataset to "mini-fit" the i-th model ds = Dataset(samples, y, None, show_log=False) + # Fiting the model with the batch model_mse += model.fit(ds, samples.size(0), 1) - model_mse/=len(batches) + model_mse /= len(batches) logger.info("MSE: %f", model_mse) - mse.append(model_mse) + + mse.append(model_mse) return mse @@ -351,7 +362,7 @@ def reconstruct( dataset (torch.utils.data.Dataset): A Dataset object containing the training data. Returns: - (Tuple[float, torch.Tensor]): Reconstruction error and visible probabilities, i.e., P(v|h). + Reconstruction error and visible probabilities, i.e., P(v|h). """ @@ -401,15 +412,17 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: x: An input tensor for computing the forward pass. Returns: - (torch.Tensor): A tensor containing the ConvDBN's outputs. + A tensor containing the ConvDBN's outputs. """ i = 0 for model in self.models: x, _ = model.hidden_sampling(x) + if self.maxpooling[i]: x = self.maxpol2d[i](x) - i+=1 + + i += 1 return x diff --git a/learnergy/models/deep/dbn.py b/learnergy/models/deep/dbn.py index 1ef5e31..65dc3e8 100644 --- a/learnergy/models/deep/dbn.py +++ b/learnergy/models/deep/dbn.py @@ -10,7 +10,7 @@ import learnergy.utils.exception as e from learnergy.core import Dataset, Model from learnergy.models.bernoulli import RBM, DropoutRBM, EDropoutRBM -from learnergy.models.extra import SigmoidRBM, SigmoidRBM4deep +from learnergy.models.extra import SigmoidRBM, SigmoidRBM4Deep from learnergy.models.gaussian import ( GaussianRBM, GaussianRBM4deep, @@ -33,7 +33,7 @@ "gaussian_relu4deep": GaussianReluRBM4deep, "gaussian_selu": GaussianSeluRBM, "sigmoid": SigmoidRBM, - "sigmoid4deep": SigmoidRBM4deep, + "sigmoid4deep": SigmoidRBM4Deep, "variance_gaussian": VarianceGaussianRBM, } @@ -49,7 +49,7 @@ class DBN(Model): def __init__( self, - model: Optional[Tuple[str, ...]] = ("gaussian", ), + model: Optional[Tuple[str, ...]] = ("gaussian",), n_visible: Optional[int] = 128, n_hidden: Optional[Tuple[int, ...]] = (128,), steps: Optional[Tuple[int, ...]] = (1,), @@ -90,12 +90,16 @@ def __init__( self.decay = decay self.T = temperature - self.models = [] - model = list(model) + if not isinstance(model, tuple): + model = (model,) if len(model) < self.n_layers: - logger.info("\n\n> Incomplete number of RBMs, adding SigmoidRBMs to fill the stack!! <\n") - for i in range(len(model)-1, self.n_layers): - model.append("sigmoid4deep") + logger.info( + "Incomplete number of RBMs. Adding SigmoidRBMs to fill the stack ..." + ) + for _ in range(len(model) - 1, self.n_layers): + model += ("sigmoid4deep",) + + self.models = [] for i in range(self.n_layers): if i == 0: n_input = self.n_visible @@ -104,38 +108,37 @@ def __init__( n_input = self.n_hidden[i - 1] if model[i] == "sigmoid": - model[i] = "sigmoid4deep" + model[i] = "sigmoid4deep" elif model[i] == "gaussian": model[i] = "gaussian4deep" elif model[i] == "gaussian_relu": model[i] = "gaussian_relu4deep" try: m = MODELS[model[i]]( - n_input, - self.n_hidden[i], - self.steps[i], - self.lr[i], - self.momentum[i], - self.decay[i], - self.T[i], - use_gpu, - normalize, - input_normalize, + n_input, + self.n_hidden[i], + self.steps[i], + self.lr[i], + self.momentum[i], + self.decay[i], + self.T[i], + use_gpu, + normalize, + input_normalize, ) - + except: m = MODELS[model[i]]( - n_input, - self.n_hidden[i], - self.steps[i], - self.lr[i], - self.momentum[i], - self.decay[i], - self.T[i], - use_gpu, + n_input, + self.n_hidden[i], + self.steps[i], + self.lr[i], + self.momentum[i], + self.decay[i], + self.T[i], + use_gpu, ) self.models.append(m) - model = tuple(model) if self.device == "cuda": self.cuda() @@ -268,7 +271,7 @@ def fit( epochs: Number of training epochs per layer. Returns: - (Tuple[float, float]): MSE (mean squared error) and log pseudo-likelihood from the training step. + MSE (mean squared error) and log pseudo-likelihood from the training step. """ @@ -277,19 +280,19 @@ def fit( mse, pl = [], [] - try: + try: samples, targets, transform = ( - dataset.data.numpy(), - dataset.targets.numpy(), - dataset.transform, + dataset.data.numpy(), + dataset.targets.numpy(), + dataset.transform, ) d = Dataset(samples, targets, transform) except: try: samples, targets, transform = ( - dataset.data, - dataset.targets, - dataset.transform, + dataset.data, + dataset.targets, + dataset.transform, ) d = Dataset(samples, targets, transform) except: @@ -299,12 +302,12 @@ def fit( for i, model in enumerate(self.models): logger.info("Fitting layer %d/%d ...", i + 1, self.n_layers) - - if i ==0: + + if i == 0: model_mse, model_pl = model.fit(d, batch_size, epochs[i]) mse.append(model_mse) pl.append(model_pl) - else: + else: # creating the training phase for deeper models for ep in range(epochs[i]): logger.info("Epoch %d/%d", ep + 1, epochs[i]) @@ -316,26 +319,24 @@ def fit( if self.device == "cuda": samples = samples.cuda() - for ii in range(i): samples, _ = self.models[ii].hidden_sampling(samples) - # Creating the dataset to ''mini-fit'' the i-th model + # Creating the dataset to ''mini-fit'' the i-th model ds = Dataset(samples, y, None, show_log=False) # Fiting the model with the batch mse_, plh = model.fit(ds, samples.size(0), 1) model_mse += mse_ pl_ += plh - model_mse/=len(batches) - pl_/=len(batches) + model_mse /= len(batches) + pl_ /= len(batches) - #logger.info("MSE: %f", model_mse) + # logger.info("MSE: %f", model_mse) logger.info("MSE: %f | log-PL: %f", model_mse, pl_) mse.append(model_mse) pl.append(pl_) - return mse, pl @@ -348,7 +349,7 @@ def reconstruct( dataset (torch.utils.data.Dataset): A Dataset object containing the training data. Returns: - (Tuple[float, torch.Tensor]): Reconstruction error and visible probabilities, i.e., P(v|h). + Reconstruction error and visible probabilities, i.e., P(v|h). """ @@ -394,7 +395,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: x: An input tensor for computing the forward pass. Returns: - (torch.Tensor): A tensor containing the DBN's outputs. + A tensor containing the DBN's outputs. """ diff --git a/learnergy/models/deep/residual_dbn.py b/learnergy/models/deep/residual_dbn.py index 669a3c4..7f372d4 100644 --- a/learnergy/models/deep/residual_dbn.py +++ b/learnergy/models/deep/residual_dbn.py @@ -48,8 +48,8 @@ def __init__( momentum (tuple): Momentum parameter per layer. decay (tuple): Weight decay used for penalization per layer. temperature (tuple): Temperature factor per layer. - zetta1 (float): Penalization factor for original learning. - zetta2 (float): Penalization factor for residual learning. + zetta1 Penalization factor for original learning. + zetta2 Penalization factor for residual learning. use_gpu (boolean): Whether GPU should be used or not. """ @@ -101,10 +101,10 @@ def calculate_residual(self, pre_activations: torch.Tensor) -> torch.Tensor: """Calculates the residual learning over input. Args: - pre_activations (torch.Tensor): Pre-activations to be used. + pre_activations Pre-activations to be used. Returns: - (torch.Tensor): The residual learning based on input pre-activations. + The residual learning based on input pre-activations. """ @@ -127,7 +127,7 @@ def fit( epochs: Number of training epochs per layer. Returns: - (Tuple[float, float]): MSE (mean squared error) and log pseudo-likelihood from the training step. + MSE (mean squared error) and log pseudo-likelihood from the training step. """ @@ -187,7 +187,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: x: An input tensor for computing the forward pass. Returns: - (torch.Tensor): A tensor containing the DBN's outputs. + A tensor containing the DBN's outputs. """ diff --git a/learnergy/models/extra/__init__.py b/learnergy/models/extra/__init__.py index 1358ae4..1a6bcea 100644 --- a/learnergy/models/extra/__init__.py +++ b/learnergy/models/extra/__init__.py @@ -1,4 +1,4 @@ """A package contaning additional RBM-based models (networks) for all common learnergy modules. """ -from learnergy.models.extra.sigmoid_rbm import SigmoidRBM, SigmoidRBM4deep +from learnergy.models.extra.sigmoid_rbm import SigmoidRBM, SigmoidRBM4Deep diff --git a/learnergy/models/extra/sigmoid_rbm.py b/learnergy/models/extra/sigmoid_rbm.py index 1ad9101..02fc07b 100644 --- a/learnergy/models/extra/sigmoid_rbm.py +++ b/learnergy/models/extra/sigmoid_rbm.py @@ -2,12 +2,10 @@ """ import time - from typing import Optional, Tuple import torch import torch.nn.functional as F - from torch.utils.data import DataLoader from learnergy.models.bernoulli import RBM @@ -76,7 +74,7 @@ def visible_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The states and probabilities of the visible layer sampling. + The states and probabilities of the visible layer sampling. """ @@ -93,7 +91,7 @@ def visible_sampling( return states, probs -class SigmoidRBM4deep(SigmoidRBM): +class SigmoidRBM4Deep(SigmoidRBM): """A SigmoidRBM class provides the basic implementation for Sigmoid-Bernoulli Restricted Boltzmann Machines. @@ -157,7 +155,7 @@ def fit( epochs: Number of training epochs. Returns: - (Tuple[float, float]): MSE (mean squared error) and log pseudo-likelihood from the training step. + MSE (mean squared error) and log pseudo-likelihood from the training step. """ @@ -206,4 +204,4 @@ def fit( self.dump(mse=mse.item(), pl=pl.item(), time=end - start) - return mse, pl \ No newline at end of file + return mse, pl diff --git a/learnergy/models/gaussian/__init__.py b/learnergy/models/gaussian/__init__.py index d0ccbc4..9599d05 100644 --- a/learnergy/models/gaussian/__init__.py +++ b/learnergy/models/gaussian/__init__.py @@ -1,7 +1,10 @@ """A package contaning gaussian-valued models (networks) for all common learnergy modules. """ -from learnergy.models.gaussian.gaussian_conv_rbm import GaussianConvRBM, GaussianConvRBM4deep +from learnergy.models.gaussian.gaussian_conv_rbm import ( + GaussianConvRBM, + GaussianConvRBM4Deep, +) from learnergy.models.gaussian.gaussian_rbm import ( GaussianRBM, GaussianRBM4deep, diff --git a/learnergy/models/gaussian/gaussian_conv_rbm.py b/learnergy/models/gaussian/gaussian_conv_rbm.py index e065c03..9dc7d50 100644 --- a/learnergy/models/gaussian/gaussian_conv_rbm.py +++ b/learnergy/models/gaussian/gaussian_conv_rbm.py @@ -94,10 +94,10 @@ def hidden_sampling(self, v: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """Performs the hidden layer sampling, i.e., P(h|v). Args: - v (torch.Tensor): A tensor incoming from the visible layer. + v A tensor incoming from the visible layer. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -113,7 +113,7 @@ def visible_sampling(self, h: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor] h: A tensor incoming from the hidden layer. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the visible layer sampling. + The probabilities and states of the visible layer sampling. """ @@ -123,7 +123,7 @@ def visible_sampling(self, h: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor] # Uses the previously calculated activations probs = activations.detach() else: - #probs = F.relu6(activations).detach() + # probs = F.relu6(activations).detach() probs = self.sig(activations).detach() return probs, probs @@ -142,7 +142,7 @@ def fit( epochs: Number of training epochs. Returns: - (float): MSE (mean squared error) from the training step. + MSE (mean squared error) from the training step. """ @@ -155,7 +155,7 @@ def fit( start = time.time() - mse = 0 + mse = 0 for samples, _ in tqdm(batches): samples = samples.reshape( @@ -200,7 +200,7 @@ def fit( return mse -class GaussianConvRBM4deep(ConvRBM): +class GaussianConvRBM4Deep(ConvRBM): """A GaussianConvRBM class provides the basic implementation for Gaussian-based Convolutional Restricted Boltzmann Machines. @@ -236,13 +236,15 @@ def __init__( learning_rate: Learning rate. momentum: Momentum parameter. decay: Weight decay used for penalization. + maxpooling: Whether MaxPooling2D should be used or not. + pooling_kernel: Pooling kernel size. use_gpu: Whether GPU should be used or not. """ - logger.info("Overriding class: ConvRBM -> GaussianConvRBM.") + logger.info("Overriding class: ConvRBM -> GaussianConvRBM4Deep.") - super(GaussianConvRBM4deep, self).__init__( + super(GaussianConvRBM4Deep, self).__init__( visible_shape, filter_shape, n_filters, @@ -258,7 +260,7 @@ def __init__( self.normalize = True - # Creating a Sigmoid function to employ on sampling + # Creates a Sigmoid function to employ on sampling self.sig = torch.nn.Sigmoid() logger.info("Class overrided.") @@ -277,10 +279,10 @@ def hidden_sampling(self, v: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """Performs the hidden layer sampling, i.e., P(h|v). Args: - v (torch.Tensor): A tensor incoming from the visible layer. + v A tensor incoming from the visible layer. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -296,7 +298,7 @@ def visible_sampling(self, h: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor] h: A tensor incoming from the hidden layer. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the visible layer sampling. + The probabilities and states of the visible layer sampling. """ @@ -307,7 +309,6 @@ def visible_sampling(self, h: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor] probs = activations.detach() else: probs = F.relu6(activations).detach() - #probs = self.sig(activations).detach() return probs, probs @@ -325,7 +326,7 @@ def fit( epochs: Number of training epochs. Returns: - (float): MSE (mean squared error) from the training step. + MSE (mean squared error) from the training step. """ @@ -334,11 +335,11 @@ def fit( ) for epoch in range(epochs): - #logger.info("Epoch %d/%d", epoch + 1, epochs) + logger.info("Epoch %d/%d", epoch + 1, epochs) start = time.time() - mse = 0 + mse = 0 for _, (samples, _) in enumerate(batches): samples = samples.reshape( @@ -378,6 +379,6 @@ def fit( self.dump(mse=mse.item(), time=end - start) - #logger.info("MSE: %f", mse) + logger.info("MSE: %f", mse) return mse diff --git a/learnergy/models/gaussian/gaussian_rbm.py b/learnergy/models/gaussian/gaussian_rbm.py index 798609c..255ff70 100644 --- a/learnergy/models/gaussian/gaussian_rbm.py +++ b/learnergy/models/gaussian/gaussian_rbm.py @@ -109,7 +109,7 @@ def energy(self, samples: torch.Tensor) -> torch.Tensor: samples: Samples to be energy-freed. Returns: - (torch.Tensor): The system's energy based on input samples. + The system's energy based on input samples. """ @@ -135,7 +135,7 @@ def visible_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the visible layer sampling. + The probabilities and states of the visible layer sampling. """ @@ -164,7 +164,7 @@ def fit( epochs: Number of training epochs. Returns: - (Tuple[float, float]): MSE (mean squared error) and log pseudo-likelihood from the training step. + MSE (mean squared error) and log pseudo-likelihood from the training step. """ @@ -233,7 +233,7 @@ def reconstruct( dataset (torch.utils.data.Dataset): A Dataset object containing the testing data. Returns: - (Tuple[float, torch.Tensor]): Reconstruction error and visible probabilities, i.e., P(v|h). + Reconstruction error and visible probabilities, i.e., P(v|h). """ @@ -278,7 +278,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: x: An input tensor for computing the forward pass. Returns: - (torch.Tensor): A tensor containing the RBM's outputs. + A tensor containing the RBM's outputs. """ @@ -360,7 +360,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -449,7 +449,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -550,7 +550,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -577,7 +577,7 @@ def visible_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the visible layer sampling. + The probabilities and states of the visible layer sampling. """ @@ -601,7 +601,7 @@ def energy(self, samples: torch.Tensor) -> torch.Tensor: samples: Samples to be energy-freed. Returns: - (torch.Tensor): The system's energy based on input samples. + The system's energy based on input samples. """ @@ -618,6 +618,7 @@ def energy(self, samples: torch.Tensor) -> torch.Tensor: return energy + class GaussianRBM4deep(GaussianRBM): """A GaussianRBM class provides the basic implementation for Gaussian-Bernoulli Restricted Boltzmann Machines (with standardization). @@ -682,7 +683,6 @@ def __init__( logger.info("Class overrided.") - def fit( self, dataset: torch.utils.data.Dataset, @@ -697,7 +697,7 @@ def fit( epochs: Number of training epochs. Returns: - (Tuple[float, float]): MSE (mean squared error) and log pseudo-likelihood from the training step. + MSE (mean squared error) and log pseudo-likelihood from the training step. """ @@ -823,7 +823,7 @@ def hidden_sampling( scale: A boolean to decide whether temperature should be used or not. Returns: - (Tuple[torch.Tensor, torch.Tensor]): The probabilities and states of the hidden layer sampling. + The probabilities and states of the hidden layer sampling. """ @@ -837,4 +837,4 @@ def hidden_sampling( # Current states equals probabilities states = probs - return probs, states \ No newline at end of file + return probs, states diff --git a/learnergy/utils/logging.py b/learnergy/utils/logging.py index bb6669a..d0d74da 100644 --- a/learnergy/utils/logging.py +++ b/learnergy/utils/logging.py @@ -14,7 +14,7 @@ def get_console_handler() -> StreamHandler: """Gets a console handler to handle logging into console. Returns: - (StreamHandler): Handler to output information into console. + Handler to output information into console. """ @@ -28,7 +28,7 @@ def get_timed_file_handler() -> TimedRotatingFileHandler: """Gets a timed file handler to handle logging into files. Returns: - (TimedRotatingFileHandler): Handler to output information into timed files. + Handler to output information into timed files. """ @@ -45,7 +45,7 @@ def get_logger(logger_name: str) -> Logger: logger_name: The name of the logger. Returns: - (Logger): Logger instance. + Logger instance. """ diff --git a/requirements.txt b/requirements.txt index bd9ee0f..429b170 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,6 @@ -coverage>=5.5 matplotlib>=3.3.4 Pillow>=8.1.2 pre-commit>=2.17.0 -pylint>=2.7.2 -pytest>=6.2.2 requests>=2.23.0 scikit-image>=0.17.2 torch>=1.8.0 diff --git a/setup.py b/setup.py index 17c4b56..23540d0 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="learnergy", - version="1.1.3", + version="1.1.4", description="Energy-based Machine Learners", long_description=long_description, long_description_content_type="text/markdown", @@ -14,12 +14,9 @@ url="https://github.com/gugarosa/learnergy", license="Apache 2.0", install_requires=[ - "coverage>=5.5", "matplotlib>=3.3.4", "Pillow>=8.1.2", "pre-commit>=2.17.0", - "pylint>=2.7.2", - "pytest>=6.2.2", "requests>2.23.0", "scikit-image>=0.17.2", "torch>=1.8.0", @@ -29,6 +26,7 @@ extras_require={ "tests": [ "coverage", + "pylint", "pytest", "pytest-pep8", ],