training-a-dnn-for-image-classification.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Training Deep Neural Networks","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-04T11:19:05.975154Z","iopub.execute_input":"2024-07-04T11:19:05.975517Z","iopub.status.idle":"2024-07-04T11:19:07.099334Z","shell.execute_reply.started":"2024-07-04T11:19:05.975487Z","shell.execute_reply":"2024-07-04T11:19:07.098173Z"}}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:49:58.311358Z","iopub.execute_input":"2024-07-04T13:49:58.311740Z","iopub.status.idle":"2024-07-04T13:50:11.237799Z","shell.execute_reply.started":"2024-07-04T13:49:58.311711Z","shell.execute_reply":"2024-07-04T13:50:11.236808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building a Deep Neural Network.","metadata":{}},{"cell_type":"code","source":"tf.random.set_seed(42)\n\nmodel = tf.keras.Sequential()\nmodel.add(tf.keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(tf.keras.layers.Dense(100, activation=\"swish\", kernel_initializer=\"he_normal\"))","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:50:11.239767Z","iopub.execute_input":"2024-07-04T13:50:11.240356Z","iopub.status.idle":"2024-07-04T13:50:11.611802Z","shell.execute_reply.started":"2024-07-04T13:50:11.240327Z","shell.execute_reply":"2024-07-04T13:50:11.610913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ading 10 nodes to the output layer as we will train the network on the CIFAR10 dataset containing 60,000 color\n# ... images with 10 classes.\n\nmodel.add(tf.keras.layers.Dense(10, activation=\"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:50:11.612984Z","iopub.execute_input":"2024-07-04T13:50:11.613397Z","iopub.status.idle":"2024-07-04T13:50:11.645594Z","shell.execute_reply.started":"2024-07-04T13:50:11.613361Z","shell.execute_reply":"2024-07-04T13:50:11.644641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Nadam Optimizer","metadata":{}},{"cell_type":"code","source":"# We'll use the optimizer with the 5e-5 learning rate.\n\noptimizer = tf.keras.optimizers.Nadam(learning_rate=5e-5)\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:50:11.647861Z","iopub.execute_input":"2024-07-04T13:50:11.648176Z","iopub.status.idle":"2024-07-04T13:50:11.663742Z","shell.execute_reply.started":"2024-07-04T13:50:11.648150Z","shell.execute_reply":"2024-07-04T13:50:11.662694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the Dataset","metadata":{}},{"cell_type":"code","source":"# We'll also create a validation set to allow us to use early stopping.\n\ncifar10 = tf.keras.datasets.cifar10.load_data()\n(X_train_full, y_train_full), (X_test, y_test) = cifar10\n\nX_train = X_train_full[5000:]\ny_train = y_train_full[5000:]\nX_valid = X_train_full[:5000]\ny_valid = y_train_full[:5000]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:50:11.665094Z","iopub.execute_input":"2024-07-04T13:50:11.665514Z","iopub.status.idle":"2024-07-04T13:50:26.171608Z","shell.execute_reply.started":"2024-07-04T13:50:11.665476Z","shell.execute_reply":"2024-07-04T13:50:26.170641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating the Necesary Callbacks.","metadata":{}},{"cell_type":"code","source":"early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=20, restore_best_weights=True)\nmodel_checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"my_cifar10_model.keras\", save_best_only=True)\n\nrun_index = 1 # Incrementing every time we train the model.\nrun_logdir = Path() / \"my_cifar10_logs\" / f\"run_{run_index:03d}\"\ntensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir)\ncallbacks = [early_stopping_cb, model_checkpoint_cb, tensorboard_cb]","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:50:26.173005Z","iopub.execute_input":"2024-07-04T13:50:26.173440Z","iopub.status.idle":"2024-07-04T13:50:26.180416Z","shell.execute_reply.started":"2024-07-04T13:50:26.173388Z","shell.execute_reply":"2024-07-04T13:50:26.179268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext tensorboard\n%tensorboard --logdir=./my_cifr10_logs","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:50:26.182305Z","iopub.execute_input":"2024-07-04T13:50:26.182750Z","iopub.status.idle":"2024-07-04T13:50:33.223513Z","shell.execute_reply.started":"2024-07-04T13:50:26.182714Z","shell.execute_reply":"2024-07-04T13:50:33.221955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the DNN","metadata":{}},{"cell_type":"code","source":"model.fit(X_train, y_train, epochs=100, validation_data=(X_valid, y_valid), callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:50:33.224902Z","iopub.execute_input":"2024-07-04T13:50:33.226385Z","iopub.status.idle":"2024-07-04T13:59:28.589012Z","shell.execute_reply.started":"2024-07-04T13:50:33.226340Z","shell.execute_reply":"2024-07-04T13:59:28.587896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:59:28.590509Z","iopub.execute_input":"2024-07-04T13:59:28.590876Z","iopub.status.idle":"2024-07-04T13:59:29.133236Z","shell.execute_reply.started":"2024-07-04T13:59:28.590849Z","shell.execute_reply":"2024-07-04T13:59:29.132149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The best model from above performs to about 44% accuracy. We'll try to improve it with Batch Normalization.","metadata":{}},{"cell_type":"markdown","source":"# Adding Batch Normalization to the DNN","metadata":{}},{"cell_type":"markdown","source":"- We will add a BN Layer after every Dense Layer(before the activation function), except for the output layer.\n- Changing the learning rate to 5e-4.\n- Also rename the run directories.","metadata":{}},{"cell_type":"code","source":"tf.random.set_seed(42)\n\nmodel = tf.keras.Sequential()\nmodel.add(tf.keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(tf.keras.layers.Dense(100, kernel_initializer=\"he_normal\"))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(tf.keras.layers.Activation(\"swish\"))\n\nmodel.add(tf.keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = tf.keras.optimizers.Nadam(learning_rate=5e-4)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=20,\n                                                     restore_best_weights=True)\nmodel_checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"my_cifar10_bn_model.keras\",\n                                                         save_best_only=True)\nrun_index = 1 # increment every time you train the model\nrun_logdir = Path() / \"my_cifar10_logs\" / f\"run_bn_{run_index:03d}\"\ntensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir)\ncallbacks = [early_stopping_cb, model_checkpoint_cb, tensorboard_cb]\n\nmodel.fit(X_train, y_train, epochs=100,\n          validation_data=(X_valid, y_valid),\n          callbacks=callbacks)\n\nmodel.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T13:59:29.136870Z","iopub.execute_input":"2024-07-04T13:59:29.137281Z","iopub.status.idle":"2024-07-04T14:08:34.397962Z","shell.execute_reply.started":"2024-07-04T13:59:29.137252Z","shell.execute_reply":"2024-07-04T14:08:34.396801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is not impressive, next we'll try replacing the batch normalization with SELU while making the necessary adjustments to ensure the network self-normalizes(use LeCun normal initializationa and ensure the DNN contains only a sequence of dense layers)","metadata":{}},{"cell_type":"code","source":"## Using SELU\n\ntf.random.set_seed(42)\n\nmodel = tf.keras.Sequential()\nmodel.add(tf.keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(tf.keras.layers.Dense(100,\n                                    kernel_initializer=\"lecun_normal\",\n                                    activation=\"selu\"))\n\nmodel.add(tf.keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = tf.keras.optimizers.Nadam(learning_rate=7e-4)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(\n    patience=20, restore_best_weights=True)\nmodel_checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\n    \"my_cifar10_selu_model.keras\", save_best_only=True)\nrun_index = 1 # increment every time you train the model\nrun_logdir = Path() / \"my_cifar10_logs\" / f\"run_selu_{run_index:03d}\"\ntensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir)\ncallbacks = [early_stopping_cb, model_checkpoint_cb, tensorboard_cb]\n\nX_means = X_train.mean(axis=0)\nX_stds = X_train.std(axis=0)\nX_train_scaled = (X_train - X_means) / X_stds\nX_valid_scaled = (X_valid - X_means) / X_stds\nX_test_scaled = (X_test - X_means) / X_stds\n\nmodel.fit(X_train_scaled, y_train, epochs=100,\n          validation_data=(X_valid_scaled, y_valid),\n          callbacks=callbacks)\n\nmodel.evaluate(X_valid_scaled, y_valid)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T14:08:34.400535Z","iopub.execute_input":"2024-07-04T14:08:34.400973Z","iopub.status.idle":"2024-07-04T14:14:27.664141Z","shell.execute_reply.started":"2024-07-04T14:08:34.400935Z","shell.execute_reply":"2024-07-04T14:14:27.662989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We can see some improvement but the network yet again started overfitting the data.**","metadata":{}},{"cell_type":"markdown","source":"# Regularizing the Model with DropOut","metadata":{}},{"cell_type":"code","source":"tf.random.set_seed(42)\n\nmodel = tf.keras.Sequential()\nmodel.add(tf.keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(tf.keras.layers.Dense(100,\n                                    kernel_initializer=\"lecun_normal\",\n                                    activation=\"selu\"))\n\nmodel.add(tf.keras.layers.Dropout(rate=0.1, noise_shape=None, seed=None))\nmodel.add(tf.keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = tf.keras.optimizers.Nadam(learning_rate=5e-4)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(\n    patience=20, restore_best_weights=True)\nmodel_checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\n    \"my_cifar10_alpha_dropout_model.keras\", save_best_only=True)\nrun_index = 1 # increment every time you train the model\nrun_logdir = Path() / \"my_cifar10_logs\" / f\"run_alpha_dropout_{run_index:03d}\"\ntensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir)\ncallbacks = [early_stopping_cb, model_checkpoint_cb, tensorboard_cb]\n\nX_means = X_train.mean(axis=0)\nX_stds = X_train.std(axis=0)\nX_train_scaled = (X_train - X_means) / X_stds\nX_valid_scaled = (X_valid - X_means) / X_stds\nX_test_scaled = (X_test - X_means) / X_stds\n\nmodel.fit(X_train_scaled, y_train, epochs=100,\n          validation_data=(X_valid_scaled, y_valid),\n          callbacks=callbacks)\n\nmodel.evaluate(X_valid_scaled, y_valid)","metadata":{"execution":{"iopub.status.busy":"2024-07-04T14:14:27.665474Z","iopub.execute_input":"2024-07-04T14:14:27.665822Z","iopub.status.idle":"2024-07-04T14:20:48.594246Z","shell.execute_reply.started":"2024-07-04T14:14:27.665794Z","shell.execute_reply":"2024-07-04T14:20:48.592899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like I'll need to upgrade to a CNN to classify images","metadata":{}},{"cell_type":"code","source":"tf.random.set_seed(42)\n\nmodel = tf.keras.Sequential()\nmodel.add(tf.keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(tf.keras.layers.Dense(100,\n                                 kernel_initializer=\"lecun_normal\",\n                                 activation=\"selu\"))\n\nmodel.add(tf.keras.layers.Dropout(rate=0.1))\nmodel.add(tf.keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = tf.keras.optimizers.SGD(learning_rate=2e-2)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2024-07-04T14:20:48.595849Z","iopub.execute_input":"2024-07-04T14:20:48.596260Z","iopub.status.idle":"2024-07-04T14:20:49.008529Z","shell.execute_reply.started":"2024-07-04T14:20:48.596228Z","shell.execute_reply":"2024-07-04T14:20:49.007525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OneCycleScheduler(tf.keras.callbacks.Callback):\n    def __init__(self, iterations, max_lr, start_lr=None, end_lr=None):\n        super().__init__()\n        self.iterations = iterations\n        self.max_lr = max_lr\n        self.start_lr = start_lr or max_lr / 10\n        self.end_lr = end_lr or self.start_lr / 1000\n        self.iteration = 0\n        self.history = {}\n\n    def on_train_begin(self, logs=None):\n        self.model.optimizer.lr = self.start_lr\n\n    def on_train_batch_begin(self, batch, logs=None):\n        if self.iteration >= self.iterations:\n            return\n        \n        cycle = np.floor(1 + self.iteration / (self.iterations / 2))\n        x = np.abs(self.iteration / (self.iterations / 2) - 2 * cycle + 1)\n        if self.iteration <= self.iterations / 2:\n            lr = self.start_lr + (self.max_lr - self.start_lr) * (1 - x)\n        else:\n            lr = self.end_lr + (self.max_lr - self.end_lr) * x\n        \n        self.model.optimizer.lr = lr\n        self.iteration += 1\n        self.history.setdefault('lr', []).append(lr)\n        self.history.setdefault('iterations', []).append(self.iteration)\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        logs['lr'] = self.model.optimizer.lr","metadata":{"execution":{"iopub.status.busy":"2024-07-04T14:20:49.009845Z","iopub.execute_input":"2024-07-04T14:20:49.010264Z","iopub.status.idle":"2024-07-04T14:20:49.022772Z","shell.execute_reply.started":"2024-07-04T14:20:49.010228Z","shell.execute_reply":"2024-07-04T14:20:49.021627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nn_epochs = 15\nbatch_size = 128\nn_iterations = math.ceil(len(X_train_scaled) / batch_size) * n_epochs\nonecycle = OneCycleScheduler(n_iterations, max_lr=0.05)\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs, batch_size=batch_size,\n                    validation_data=(X_valid_scaled, y_valid),\n                    callbacks=[onecycle])","metadata":{"execution":{"iopub.status.busy":"2024-07-04T14:28:20.193788Z","iopub.execute_input":"2024-07-04T14:28:20.194228Z","iopub.status.idle":"2024-07-04T14:29:19.999656Z","shell.execute_reply.started":"2024-07-04T14:28:20.194192Z","shell.execute_reply":"2024-07-04T14:29:19.998510Z"},"trusted":true},"execution_count":null,"outputs":[]}]}