MLP/mnist_sigmoid.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import gzip\n",
    "import pickle\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorized_result(j):\n",
    "    \"\"\"Return a 10-dimensional unit vector with a 1.0 in the jth\n",
    "    position and zeroes elsewhere.  This is used to convert a digit\n",
    "    (0...9) into a corresponding desired output from the neural\n",
    "    network.\"\"\"\n",
    "    e = np.zeros((10, 1))\n",
    "    e[j] = 1.0\n",
    "    return e\n",
    "\n",
    "\n",
    "def backprop(x, y, biases, weights, num_layers):\n",
    "    \"\"\"Return a tuple ``(nabla_b, nabla_w)`` representing the\n",
    "    gradient for the cost function C_x.  ``nabla_b`` and\n",
    "    ``nabla_w`` are layer-by-layer lists of numpy arrays, similar\n",
    "    to ``self.biases`` and ``self.weights``.\"\"\"\n",
    "    \n",
    "    # 初始化空偏置值集合和空权重集合\n",
    "    nabla_b = [np.zeros(b.shape) for b in biases]\n",
    "    nabla_w = [np.zeros(w.shape) for w in weights]\n",
    "    \n",
    "    # feedforward前馈\n",
    "    # x为单一训练样本\n",
    "    activation = x\n",
    "    activations = [x] # list to store all the activations, layer by layer\n",
    "    # 存储所有z向量集合 z = (a * w + b)\n",
    "    zs = [] # list to store all the z vectors, layer by layer\n",
    "    \n",
    "    # 用初始化的随机偏置集和权重集逐层进行计算\n",
    "    for b, w in list(zip(biases, weights)):\n",
    "        z = np.dot(w, activation)+b\n",
    "        # 记录了神经网络中每层的计算结果的值\n",
    "        zs.append(z)\n",
    "        \n",
    "        # 为了理解和感知器模型的相似性，假设z ≡ w ·x + b 是一个很大的正数。那么e−z≈ 0 而\n",
    "        # σ(z) ≈1。即，当z = w ·x + b 很大并且为正，S型神经元的输出近似为1，正好和感知器一样。\n",
    "        # 相反地，假设z = w ·x + b 是一个很大的负数。那么e−z→∞，σ(z) ≈0。所以当z = w ·x + b\n",
    "        # 是一个很大的负数，S型神经元的行为也非常近似一个感知器。只有在w ·x + b 取中间值时，和感知器模型有比较大的偏离。\n",
    "        \n",
    "        # 计算每层激活值\n",
    "        activation = sigmoid(z)\n",
    "        # 记录每层神经元的激活值\n",
    "        activations.append(activation)\n",
    "        \n",
    "    # backward pass 反向传播\n",
    "    # sigmoid_prime 是 sigmoid方程的求导，表示了对结果影响的程度大小\n",
    "    # cost_derivative 求出计算结果与训练集中给出对应结果的差值\n",
    "    # delta 表示这个偏差对结果影响的程度的值\n",
    "    delta = cost_derivative(activations[-1], y) * sigmoid_prime(zs[-1])\n",
    "    # 这个值可以直接在偏置量上进行加减运算，即偏置量的梯度\n",
    "    nabla_b[-1] = delta\n",
    "    # 求权重的梯度\n",
    "    nabla_w[-1] = np.dot(delta, activations[-2].transpose())\n",
    "    # Note that the variable l in the loop below is used a little\n",
    "    # differently to the notation in Chapter 2 of the book.  Here,\n",
    "    # l = 1 means the last layer of neurons, l = 2 is the\n",
    "    # second-last layer, and so on.  It's a renumbering of the\n",
    "    # scheme in the book, used here to take advantage of the fact\n",
    "    # that Python can use negative indices in lists.\n",
    "    \n",
    "    # l = 1 表示最后一层神经元，l = 2 是倒数第二层神经元, 依此类推.\n",
    "    for l in range(2, num_layers):\n",
    "        z = zs[-l]\n",
    "        sp = sigmoid_prime(z)\n",
    "        delta = np.dot(weights[-l+1].transpose(), delta) * sp\n",
    "        nabla_b[-l] = delta\n",
    "        nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())\n",
    "    return (nabla_b, nabla_w)\n",
    "\n",
    "\n",
    "def evaluate(test_data, biases, weights):\n",
    "    \"\"\"Return the number of test inputs for which the neural\n",
    "    network outputs the correct result. Note that the neural\n",
    "    network's output is assumed to be the index of whichever\n",
    "    neuron in the final layer has the highest activation.\"\"\"\n",
    "    test_results = [(np.argmax(feedforward(x, biases, weights)), y)\n",
    "                    for (x, y) in test_data]\n",
    "    return sum(int(x == y) for (x, y) in test_results)\n",
    "\n",
    "def cost_derivative(output_activations, y):\n",
    "    \"\"\"Return the vector of partial derivatives \\partial C_x /\n",
    "    \\partial a for the output activations.\"\"\"\n",
    "    return (output_activations-y)\n",
    "\n",
    "#### Miscellaneous functions\n",
    "def sigmoid(z):\n",
    "    \"\"\"The sigmoid function.\"\"\"\n",
    "    return 1.0/(1.0+np.exp(-z))\n",
    "\n",
    "def sigmoid_prime(z):\n",
    "    \"\"\"Derivative of the sigmoid function.\"\"\"\n",
    "    return sigmoid(z)*(1-sigmoid(z))\n",
    "\n",
    "def feedforward(a, biases, weights):\n",
    "    \"\"\"Return the output of the network if ``a`` is input.\"\"\"\n",
    "    for b, w in zip(biases, weights):\n",
    "        a = sigmoid(np.dot(w, a)+b)\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1)\n",
    "\n",
    "# load data\n",
    "f = gzip.open('../data_sets/mnist.pkl.gz', \"rb\")  \n",
    "training_data, validation_data, test_data = pickle.load(f,encoding='iso-8859-1')  \n",
    "f.close()  \n",
    "\n",
    "# 训练数据集由shape(50000,784)转换成len=50000,shape(784,1)的列表\n",
    "training_inputs = [np.reshape(x, (784, 1)) for x in training_data[0]]\n",
    "# 此步骤将数据集对应结果由shape(50000,)转换成len=50000,shape(10,1)的列表，数组中的值为1/0\n",
    "training_results = [vectorized_result(y) for y in training_data[1]]\n",
    "# 此步骤将数据集与结果整合为一个数组，数组格式为len=50000的列表，每项为len=2的列表，第一项为数据集第一项shape(784,1)，第二项为结果标识shape(10,1)\n",
    "vtraining_data = list(zip(training_inputs, training_results))\n",
    "\n",
    "# 验证数据集\n",
    "validation_inputs = [np.reshape(x, (784, 1)) for x in validation_data[0]]\n",
    "validation_data = list(zip(validation_inputs, validation_data[1]))\n",
    "\n",
    "# 测试数据集\n",
    "test_inputs = [np.reshape(x, (784, 1)) for x in test_data[0]]\n",
    "test_data = list(zip(test_inputs, test_data[1]))\n",
    "\n",
    "print(len(training_inputs))\n",
    "print(len(validation_inputs))\n",
    "print(len(test_inputs))\n",
    " \n",
    "# 显示数据集图片\n",
    "img = training_inputs[0]\n",
    "# img = validation_inputs[0]\n",
    "# img = training_inputs[1]\n",
    "img = img.reshape(28,-1)\n",
    "print(type(img))\n",
    "\n",
    "# 设置图形\n",
    "plt.rcParams['figure.figsize'] = (5.0, 4.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "plt.imshow(img)\n",
    "\n",
    "# 创建三层神经网络，第一层（输入层）784个神经元，第二层（隐藏层）30个神经元，第三层（输出层）10个神经元\n",
    "sizes = [784,30,10]\n",
    "num_layers = len(sizes)\n",
    "# 为第一层和第二层神经元设置偏置值，列表len=2，第一项为第一层偏置值shape(30,1)，第二项为第二层偏置值shape(10,1)，并填充均值为0，标准差为1的高斯分布随机数\n",
    "biases = [np.random.randn(y, 1) for y in sizes[1:]]\n",
    "# 为第一层和第二层神经元设置权重，列表len=2，第一项为第一层权重shape(30,784)，第二项为第二层权重shape(10,30)，并填充均值为0，标准差为1的高斯分布随机数\n",
    "weights = [np.random.randn(y, x) for x, y in zip(sizes[:-1], sizes[1:])]\n",
    "\n",
    "# print(sizes, num_layers, biases, weights)\n",
    "\n",
    "# 训练总次数\n",
    "epochs = 30\n",
    "# 最每次训练小样本集大小\n",
    "mini_batch_size = 10\n",
    "# 学习率\n",
    "eta = 0.1\n",
    "\n",
    "# 测试数据集不为空，则获取测试数据集数量\n",
    "if test_data: \n",
    "    n_test = len(test_data)\n",
    "# vtraining_data = vtraining_data[0:1000]\n",
    "# 获取训练数据集数量\n",
    "n = len(vtraining_data)\n",
    "# 开始训练\n",
    "for j in range(epochs):\n",
    "    # 打乱训练数据集顺序\n",
    "    random.shuffle(vtraining_data)\n",
    "    # 组装每次训练数据集\n",
    "    mini_batches = [\n",
    "        vtraining_data[k:k+mini_batch_size]\n",
    "        for k in range(0, n, mini_batch_size)]\n",
    "    # 开始训练\n",
    "    for mini_batch in mini_batches:\n",
    "        # print(mini_batch)\n",
    "        \"\"\"Update the network's weights and biases by applying\n",
    "        gradient descent using backpropagation to a single mini batch.\n",
    "        The ``mini_batch`` is a list of tuples ``(x, y)``, and ``eta``\n",
    "        is the lwearning rate.\"\"\"\n",
    "        # 初始化反向传播偏置值集合\n",
    "        nabla_b = [np.zeros(b.shape) for b in biases]\n",
    "        # 初始化反向传播权重集合\n",
    "        nabla_w = [np.zeros(w.shape) for w in weights]\n",
    "        for x, y in mini_batch:\n",
    "            # 传入训练数据，累积每次计算偏移量\n",
    "            delta_nabla_b, delta_nabla_w = backprop(x, y, biases, weights, num_layers  )\n",
    "            nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]\n",
    "            nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]\n",
    "        # 定义学习率计算累积偏移量的影响，并用当前偏移量减去影响实现梯度下降\n",
    "        weights = [w-(eta/len(mini_batch))*nw for w, nw in zip(weights, nabla_w)]\n",
    "        biases = [b-(eta/len(mini_batch))*nb for b, nb in zip(biases, nabla_b)]\n",
    "        \n",
    "        # 添加统计图表数据\n",
    "        # nabla_ws.append(nabla_w[-1][0])\n",
    "    # 显示图表并清空统计数据\n",
    "    # matrix = np.full((30, len(nabla_ws)), np.arange(len(nabla_ws)))\n",
    "    # plt.figure(figsize=(30, 12))\n",
    "    # plt.plot(matrix.T,np.array(nabla_ws))\n",
    "    # plt.show()\n",
    "    # nabla_ws.clear()\n",
    "    \n",
    "    if test_data:\n",
    "        print (\"Epoch {0}: {1} / {2}\".format(\n",
    "        j, evaluate(test_data, biases, weights), n_test))\n",
    "    else:\n",
    "        print (\"Epoch {0} complete\".format(j))\n",
    "        \n",
    "# matrix = np.full((30, len(epoch_w)), np.arange(len(epoch_w)))\n",
    "# plt.figure(figsize=(30, 12))\n",
    "# plt.plot(matrix.T,np.array(nabla_ws))\n",
    "# plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}