Merge pull request #3 from Mateusmsouza/dev

Dev

README.md

@@ -0,0 +1 @@
+Yes

documentation/docs/Documentation/API-Reference.md

@@ -0,0 +1,59 @@
+<a id="nn.loss"></a>
+
+# nn.loss
+
+<a id="nn.loss.MeanSquaredError"></a>
+
+## MeanSquaredError Objects
+
+```python
+class MeanSquaredError()
+```
+
+Class to compute the Mean Squared Error (MSE) and its gradient.
+
+<a id="nn.loss.MeanSquaredError.forward"></a>
+
+#### forward
+
+```python
+def forward(y_true: np.ndarray, y_pred: np.ndarray) -> np.ndarray
+```
+
+Compute the Mean Squared Error between true and predicted values.
+
+**Arguments**:
+
+- `y_true` _np.ndarray_ - True values.
+- `y_pred` _np.ndarray_ - Predicted values.
+
+
+**Returns**:
+
+- `np.ndarray` - The mean squared error.
+
+
+**Raises**:
+
+- `ValueError` - If y_true and y_pred do not have the same shape.
+
+<a id="nn.loss.MeanSquaredError.backward"></a>
+
+#### backward
+
+```python
+def backward(y_true: np.ndarray, y_pred: np.ndarray) -> np.ndarray
+```
+
+Compute the gradient of the Mean Squared Error with respect to the predicted values.
+
+**Arguments**:
+
+- `y_true` _np.ndarray_ - True values.
+- `y_pred` _np.ndarray_ - Predicted values.
+
+
+**Returns**:
+
+- `np.ndarray` - The gradient of the loss with respect to y_pred.
+

documentation/docs/Documentation/Install.md

@@ -0,0 +1,31 @@
+# Installation
+
+## Introduction
+
+`liltorch` is a library for building and training machine learning models. This tutorial will guide you through the process of installing `liltorch` from PyPI, the Python Package Index.
+
+## Prerequisites
+
+Before installing `liltorch`, ensure that you have the following prerequisites:
+
+- Python 3.6 or higher
+- pip (Python package installer)
+
+You can verify your Python and pip versions by running the following commands in your terminal or command prompt:
+
+```sh
+python --version
+pip --version
+```
+
+## Installation Steps
+Open the terminal and type:
+```sh
+pip install --upgrade pip
+pip install liltorch
+```
+Check installation:
+```sh
+import liltorch
+print("liltorch version:", liltorch.__version__)
+```

documentation/docs/index.md

@@ -2,4 +2,4 @@
 
 LilTorch is a lightweight library for Deep Learning, created entirely in Python with Numpy as its sole dependency. This library allows you to design and understand the internals of Neural Networks without the need for C/C++ imported code and binaries. Everything is as understandable as Python itself.
 
-**Note**: This library is intended for educational purposes only. It is not recommended for production use, and execution speed is not a primary focus.
+**Note**: This library is intended for educational purposes only. It is not recommended for production use, and execution speed is not a primary focus.

documentation/mkdocs.yml

@@ -1 +1 @@
-site_name: LilTorch
+site_name: LilTorch

examples/mnist.py

@@ -7,16 +7,16 @@
 from keras.utils import to_categorical
 
 from liltorch.nn.fully_connected import FullyConnectedLayer
-from liltorch.nn.activation import ActivationLayerTanh
+from liltorch.nn.activation import Tanh
 from liltorch.nn.network import Network
 
-from liltorch.nn.loss import mse_loss, mse_grad
+from liltorch.nn.loss import MeanSquaredError
 import numpy as np
 
 # load MNIST from server
 lr = 0.1
-epochs = 5
-batch_size = 16
+epochs = 20
+batch_size = 4
 (x_train, y_train), (x_test, y_test) = mnist.load_data()
 
 x_train = x_train.reshape(x_train.shape[0], 28*28)
@@ -30,55 +30,46 @@
 y_test = to_categorical(y_test)
 
 print(f'shape of xtrain {x_train.shape} xtrain {y_train.shape} and xtrain {x_test.shape} xtest {x_test.shape}')
+
 # build model
 model = Network(lr=lr)
 model.add(FullyConnectedLayer(28*28, 100))
-model.add(ActivationLayerTanh())
+model.add(Tanh())
 model.add(FullyConnectedLayer(100, 50))
-model.add(ActivationLayerTanh())
+model.add(Tanh())
 model.add(FullyConnectedLayer(50, 10))
-model.add(ActivationLayerTanh())
+model.add(Tanh())
 
 # training
 for epoch in range(epochs):
     epoch_loss = 0
     dataset_size = len(x_train)
-    batch_begin = 0
-    batch_end = batch_size - 1
-    while batch_begin < dataset_size:
+    correct = 0
+    criterion = MeanSquaredError()
+    for batch_begin in range(0, dataset_size, batch_size):
+        batch_end = min(batch_begin + batch_size, dataset_size)
         data = x_train[batch_begin:batch_end]
         target = y_train[batch_begin:batch_end]
-        #print(f"batch from {batch_begin} to {batch_end}")
 
         # forward
         output = model.forward(data)
-        epoch_loss += mse_loss(target, output)
+        epoch_loss += criterion.foward(target, output)
 
         # backward pass
-        error = mse_grad(target, output)
+        error = criterion.backward(target, output)
         model.backward(error)
-
-        batch_begin = batch_end
-        batch_end += batch_size
-        batch_end = min(dataset_size, batch_end)
-
-    print(f'Epoch {epoch} -> Average loss {epoch_loss/len(x_train)}')
+        correct += np.sum((np.argmax(output, axis=1) == np.argmax(target, axis=1)))
+    print(f'Epoch {epoch} -> Average loss {epoch_loss/dataset_size} / Average Accuracy {(correct/dataset_size):.4f}')
 
 # testing
 correct = 0
 dataset_size = len(x_test)
-batch_begin = 0
-batch_end = batch_size - 1
-
-while batch_begin < dataset_size -1:
+for batch_begin in range(0, dataset_size, batch_size):
+    batch_end = min(batch_begin + batch_size, dataset_size)
     data = x_test[batch_begin:batch_end]
     target = y_test[batch_begin:batch_end]
-    #print(f"batch from {batch_begin} to {batch_end}")
-    output = model.forward(data)
-    correct += np.sum((np.argmax(output, axis=1) == np.argmax(target)))
 
-    batch_begin = batch_end
-    batch_end += batch_size
-    batch_end = min(dataset_size, batch_end)
+    output = model.forward(data)
+    correct += np.sum((np.argmax(output, axis=1) == np.argmax(target, axis=1)))
 
-print(f'Test Accuracy: {correct/dataset_size}')
+print(f'Test Accuracy: {(correct/dataset_size):.4f}')

liltorch/nn/activation.py

@@ -3,7 +3,7 @@
 from liltorch.nn.layer import Layer
 
 
-class ActivationLayerTanh(Layer):
+class Tanh(Layer):
 
     def forward(self, input_data):
         '''fordward pass using tanh activation'''

liltorch/nn/fully_connected.py

@@ -1,5 +1,3 @@
-import random
-
 import numpy as np
 
 from liltorch.nn.layer import Layer
@@ -17,10 +15,17 @@ def forward(self, input_data):
         self.output = np.dot(self.input, self.weights) + self.bias
         return self.output
 
-    def backward(self, output_error, lr):
-        input_error = np.dot(output_error, self.weights.T)
-        weights_error = np.dot(self.input.T, output_error)
-
-        self.weights -= lr * weights_error
-        self.bias -= lr * np.sum(output_error, axis=0, keepdims=True)
-        return input_error
+    def backward(self, upstream_gradients, lr):
+        # Calculate gradients to propagate to the previous layer (dL/dz[i]) given 
+        # a previous layer gradient (dL/dz[i+1]) (which in forward pass would be next layer)
+        downstream_gradients = np.dot(upstream_gradients, self.weights.T)
+
+        # Calculate local gradients for weights and biases (dL/dW and dL/dB )
+        local_gradients_w = np.dot(self.input.T, upstream_gradients)
+        local_gradients_b = np.sum(upstream_gradients, axis=0, keepdims=True)
+
+        # Update weights and biases using the gradients and learning rate
+        self.weights -= lr * local_gradients_w
+        self.bias -= lr * local_gradients_b
+
+        return downstream_gradients

liltorch/nn/loss.py

@@ -1,10 +1,37 @@
 import numpy as np
 
+class MeanSquaredError:
+    """
+    Class to compute the Mean Squared Error (MSE) and its gradient.
+    """
 
-def mse_loss(y_true: np.ndarray, y_pred: np.ndarray):
-    if y_true.shape != y_pred.shape:
-        raise ValueError("y_true and y_pred must have the same length.")
-    return np.mean(np.power(y_true-y_pred, 2));
+    def forward(self, y_true: np.ndarray, y_pred: np.ndarray) -> np.ndarray:
+        """
+        Compute the Mean Squared Error between true and predicted values.
 
-def mse_grad(y_true, y_pred):
-    return 2*(y_pred-y_true)/y_true.size;
+        Parameters:
+        y_true (np.ndarray): True values.
+        y_pred (np.ndarray): Predicted values.
+
+        Returns:
+        np.ndarray: The mean squared error.
+
+        Raises:
+        ValueError: If y_true and y_pred do not have the same shape.
+        """
+        if y_true.shape != y_pred.shape:
+            raise ValueError("y_true and y_pred must have the same length.")
+        return np.mean(np.power(y_true - y_pred, 2))
+
+    def backward(self, y_true: np.ndarray, y_pred: np.ndarray) -> np.ndarray:
+        """
+        Compute the gradient of the Mean Squared Error with respect to the predicted values.
+
+        Parameters:
+        y_true (np.ndarray): True values.
+        y_pred (np.ndarray): Predicted values.
+
+        Returns:
+        np.ndarray: The gradient of the loss with respect to y_pred.
+        """
+        return 2 * (y_pred - y_true) / y_true.size

requirements.dev.txt

@@ -1,3 +1,4 @@
 pytest==8.2.2
-mkdocs==1.6.0
-coverage==7.5.4
+Sphinx==7.3.7
+sphinx-autodoc-typehints==2.2.2
+sphinx-markdown-builder==0.6.6

setup.py

@@ -6,4 +6,6 @@
     description='Small neural network library made only with raw python',
     author='Mateus Souza',
     packages=find_packages(),
+    long_description=open('README.md').read(),
+    long_description_content_type='text/markdown',
 )