3.3.线性回归的简洁实现.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 生成数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:36.502793Z",
     "start_time": "2019-05-09T09:51:34.503881Z"
    }
   },
   "outputs": [],
   "source": [
    "# 与上一节相同\n",
    "from mxnet import autograd, nd\n",
    "\n",
    "num_inputs = 2\n",
    "num_examples = 1000\n",
    "true_w = [2, -3.4]\n",
    "true_b = 4.2\n",
    "features = nd.random.normal(scale=1, shape=(num_examples, num_inputs))\n",
    "labels = true_w[0] * features[:, 0] + true_w[1] * features[:, 1] + true_b\n",
    "labels += nd.random.normal(scale=0.01, shape=labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:36.509746Z",
     "start_time": "2019-05-09T09:51:36.504756Z"
    }
   },
   "outputs": [],
   "source": [
    "from mxnet.gluon import data as gdata\n",
    "\n",
    "batch_size = 10\n",
    "# 将训练数据的特征和标签组合\n",
    "dataset = gdata.ArrayDataset(features, labels)\n",
    "# 随机读取小批量\n",
    "# 这里data_iter的使用跟上一节中的一样。让我们读取并打印第一个小批量数据样本\n",
    "data_iter = gdata.DataLoader(dataset, batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:36.529689Z",
     "start_time": "2019-05-09T09:51:36.512734Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[[ 0.60375047 -0.5491552 ]\n",
      " [ 1.5329047   0.8233045 ]\n",
      " [-1.3713115   0.44100854]\n",
      " [-0.5489446   0.9783404 ]\n",
      " [ 0.61842227  0.0640344 ]\n",
      " [-0.8622762  -0.22448552]\n",
      " [-0.8627405  -1.9521301 ]\n",
      " [-2.0252197   0.14331104]\n",
      " [-0.5558463  -0.440662  ]\n",
      " [-0.9930084   1.0062088 ]]\n",
      "<NDArray 10x2 @cpu(0)> \n",
      "[ 7.2679596   4.480722   -0.02090577 -0.22527038  5.2208033   3.2509205\n",
      "  9.108723   -0.34202975  4.5954723  -1.2033253 ]\n",
      "<NDArray 10 @cpu(0)>\n"
     ]
    }
   ],
   "source": [
    "for X, y in data_iter:\n",
    "    print(X, y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:36.546641Z",
     "start_time": "2019-05-09T09:51:36.533680Z"
    }
   },
   "outputs": [],
   "source": [
    "# 准备工作\n",
    "from mxnet.gluon import nn\n",
    "from mxnet import init\n",
    "from mxnet.gluon import loss as gloss\n",
    "from mxnet import gluon\n",
    "# 实例化Sequential\n",
    "net = nn.Sequential()\n",
    "# 添加一个全连接层，输出个数为1\n",
    "net.add(nn.Dense(1))\n",
    "# 初始化时随机采样于均值为0、标准差为0.01的正态分布。偏差参数默认会初始化为零\n",
    "net.initialize(init.Normal(sigma=0.01))\n",
    "# 平方损失又称L2范数损失\n",
    "loss = gloss.L2Loss()\n",
    "# 创建一个Trainer实例，并指定学习率为0.03的小批量随机梯度下降（sgd）为优化算法\n",
    "# 该优化算法将用来迭代net实例所有通过add函数嵌套的层所包含的全部参数。\n",
    "# 这些参数可以通过collect_params函数获取\n",
    "trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.03})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:36.947571Z",
     "start_time": "2019-05-09T09:51:36.548636Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, loss: 0.035009\n",
      "epoch 2, loss: 0.000129\n",
      "epoch 3, loss: 0.000049\n"
     ]
    }
   ],
   "source": [
    "# 训练模型\n",
    "# 由于变量l是长度为batch_size的一维NDArray，执行l.backward()等价于执行l.sum().backward()。\n",
    "# 按照小批量随机梯度下降的定义，我们在step函数中指明批量大小，从而对批量中样本梯度求平均\n",
    "num_epochs = 3\n",
    "for epoch in range(1, num_epochs + 1):\n",
    "    for X, y in data_iter:\n",
    "        with autograd.record():\n",
    "            l = loss(net(X), y)\n",
    "        l.backward()\n",
    "        trainer.step(batch_size) # 与自己写的唯一区别在这步，自己写这里是sgd\n",
    "    l = loss(net(features), labels)\n",
    "    print('epoch %d, loss: %f' % (epoch, l.mean().asnumpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:36.968514Z",
     "start_time": "2019-05-09T09:51:36.950561Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([2, -3.4], \n",
       " [[ 1.9997326 -3.3994465]]\n",
       " <NDArray 1x2 @cpu(0)>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dense = net[0]\n",
    "true_w, dense.weight.data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:36.992449Z",
     "start_time": "2019-05-09T09:51:36.971506Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4.2, \n",
       " [4.1996355]\n",
       " <NDArray 1 @cpu(0)>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "true_b, dense.bias.data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 练习\n",
    "\n",
    "* 如果将`l = loss(net(X), y)`替换成`l = loss(net(X), y).mean()`，我们需要将`trainer.step(batch_size)`相应地改成`trainer.step(1)`。这是为什么呢？\n",
    "* 答：自动求梯度模块计算得来的梯度是一个批量样本的梯度和，所以在更新w,b的时候，因为w,b是向量，把每一个权重用总梯度更新肯定不对，所以除以batch_size，用梯度的均值来更新才对，但是如果损失函数就已经求均值了(本来输出是(num_examples,1)的向量)，在求损失函数关于w,b的梯度的时候，得到的就不是梯度和，而是均值的梯度，就不用再除batch_size了，效果一样\n",
    "* 查阅MXNet文档，看看`gluon.loss`和`init`模块里提供了哪些损失函数和初始化方法。\n",
    "* 如何访问`dense.weight`的梯度？\n",
    "dense.weight.grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:37.013399Z",
     "start_time": "2019-05-09T09:51:36.996439Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method step in module mxnet.gluon.trainer:\n",
      "\n",
      "step(batch_size, ignore_stale_grad=False) method of mxnet.gluon.trainer.Trainer instance\n",
      "    Makes one step of parameter update. Should be called after\n",
      "    `autograd.backward()` and outside of `record()` scope.\n",
      "    \n",
      "    For normal parameter updates, `step()` should be used, which internally calls\n",
      "    `allreduce_grads()` and then `update()`. However, if you need to get the reduced\n",
      "    gradients to perform certain transformation, such as in gradient clipping, then\n",
      "    you may want to manually call `allreduce_grads()` and `update()` separately.\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    batch_size : int\n",
      "        Batch size of data processed. Gradient will be normalized by `1/batch_size`.\n",
      "        Set this to 1 if you normalized loss manually with `loss = mean(loss)`.\n",
      "    ignore_stale_grad : bool, optional, default=False\n",
      "        If true, ignores Parameters with stale gradient (gradient that has not\n",
      "        been updated by `backward` after last step) and skip update.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(trainer.step)\n",
    "# Gradient will be normalized by `1/batch_size`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-09T09:51:37.031346Z",
     "start_time": "2019-05-09T09:51:37.019379Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on Parameter in module mxnet.gluon.parameter object:\n",
      "\n",
      "class Parameter(builtins.object)\n",
      " |  A Container holding parameters (weights) of Blocks.\n",
      " |  \n",
      " |  :py:class:`Parameter` holds a copy of the parameter on each :py:class:`Context` after\n",
      " |  it is initialized with ``Parameter.initialize(...)``. If :py:attr:`grad_req` is\n",
      " |  not ``'null'``, it will also hold a gradient array on each :py:class:`Context`::\n",
      " |  \n",
      " |      ctx = mx.gpu(0)\n",
      " |      x = mx.nd.zeros((16, 100), ctx=ctx)\n",
      " |      w = mx.gluon.Parameter('fc_weight', shape=(64, 100), init=mx.init.Xavier())\n",
      " |      b = mx.gluon.Parameter('fc_bias', shape=(64,), init=mx.init.Zero())\n",
      " |      w.initialize(ctx=ctx)\n",
      " |      b.initialize(ctx=ctx)\n",
      " |      out = mx.nd.FullyConnected(x, w.data(ctx), b.data(ctx), num_hidden=64)\n",
      " |  \n",
      " |  Parameters\n",
      " |  ----------\n",
      " |  name : str\n",
      " |      Name of this parameter.\n",
      " |  grad_req : {'write', 'add', 'null'}, default 'write'\n",
      " |      Specifies how to update gradient to grad arrays.\n",
      " |  \n",
      " |      - ``'write'`` means everytime gradient is written to grad :py:class:`NDArray`.\n",
      " |      - ``'add'`` means everytime gradient is added to the grad :py:class:`NDArray`. You need\n",
      " |        to manually call ``zero_grad()`` to clear the gradient buffer before each\n",
      " |        iteration when using this option.\n",
      " |      - 'null' means gradient is not requested for this parameter. gradient arrays\n",
      " |        will not be allocated.\n",
      " |  shape : int or tuple of int, default None\n",
      " |      Shape of this parameter. By default shape is not specified. Parameter with\n",
      " |      unknown shape can be used for :py:class:`Symbol` API, but ``init`` will throw an error\n",
      " |      when using :py:class:`NDArray` API.\n",
      " |  dtype : numpy.dtype or str, default 'float32'\n",
      " |      Data type of this parameter. For example, ``numpy.float32`` or ``'float32'``.\n",
      " |  lr_mult : float, default 1.0\n",
      " |      Learning rate multiplier. Learning rate will be multiplied by lr_mult\n",
      " |      when updating this parameter with optimizer.\n",
      " |  wd_mult : float, default 1.0\n",
      " |      Weight decay multiplier (L2 regularizer coefficient). Works similar to lr_mult.\n",
      " |  init : Initializer, default None\n",
      " |      Initializer of this parameter. Will use the global initializer by default.\n",
      " |  stype: {'default', 'row_sparse', 'csr'}, defaults to 'default'.\n",
      " |      The storage type of the parameter.\n",
      " |  grad_stype: {'default', 'row_sparse', 'csr'}, defaults to 'default'.\n",
      " |      The storage type of the parameter's gradient.\n",
      " |  \n",
      " |  Attributes\n",
      " |  ----------\n",
      " |  grad_req : {'write', 'add', 'null'}\n",
      " |      This can be set before or after initialization. Setting ``grad_req`` to ``'null'``\n",
      " |      with ``x.grad_req = 'null'`` saves memory and computation when you don't\n",
      " |      need gradient w.r.t x.\n",
      " |  lr_mult : float\n",
      " |      Local learning rate multiplier for this Parameter. The actual learning rate\n",
      " |      is calculated with ``learning_rate * lr_mult``. You can set it with\n",
      " |      ``param.lr_mult = 2.0``\n",
      " |  wd_mult : float\n",
      " |      Local weight decay multiplier for this Parameter.\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __init__(self, name, grad_req='write', shape=None, dtype=<class 'numpy.float32'>, lr_mult=1.0, wd_mult=1.0, init=None, allow_deferred_init=False, differentiable=True, stype='default', grad_stype='default')\n",
      " |      Initialize self.  See help(type(self)) for accurate signature.\n",
      " |  \n",
      " |  __repr__(self)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  cast(self, dtype)\n",
      " |      Cast data and gradient of this Parameter to a new data type.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      dtype : str or numpy.dtype\n",
      " |          The new data type.\n",
      " |  \n",
      " |  data(self, ctx=None)\n",
      " |      Returns a copy of this parameter on one context. Must have been\n",
      " |      initialized on this context before. For sparse parameters, use\n",
      " |      :py:meth:`Parameter.row_sparse_data` instead.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      ctx : Context\n",
      " |          Desired context.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      NDArray on ctx\n",
      " |  \n",
      " |  grad(self, ctx=None)\n",
      " |      Returns a gradient buffer for this parameter on one context.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      ctx : Context\n",
      " |          Desired context.\n",
      " |  \n",
      " |  initialize(self, init=None, ctx=None, default_init=<mxnet.initializer.Uniform object at 0x000001C6D9F6D940>, force_reinit=False)\n",
      " |      Initializes parameter and gradient arrays. Only used for :py:class:`NDArray` API.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      init : Initializer\n",
      " |          The initializer to use. Overrides :py:meth:`Parameter.init` and default_init.\n",
      " |      ctx : Context or list of Context, defaults to :py:meth:`context.current_context()`.\n",
      " |          Initialize Parameter on given context. If ctx is a list of Context, a\n",
      " |          copy will be made for each context.\n",
      " |      \n",
      " |          .. note::\n",
      " |              Copies are independent arrays. User is responsible for keeping\n",
      " |              their values consistent when updating.\n",
      " |              Normally :py:class:`gluon.Trainer` does this for you.\n",
      " |      \n",
      " |      default_init : Initializer\n",
      " |          Default initializer is used when both :py:func:`init`\n",
      " |          and :py:meth:`Parameter.init` are ``None``.\n",
      " |      force_reinit : bool, default False\n",
      " |          Whether to force re-initialization if parameter is already initialized.\n",
      " |      \n",
      " |      Examples\n",
      " |      --------\n",
      " |      >>> weight = mx.gluon.Parameter('weight', shape=(2, 2))\n",
      " |      >>> weight.initialize(ctx=mx.cpu(0))\n",
      " |      >>> weight.data()\n",
      " |      [[-0.01068833  0.01729892]\n",
      " |       [ 0.02042518 -0.01618656]]\n",
      " |      <NDArray 2x2 @cpu(0)>\n",
      " |      >>> weight.grad()\n",
      " |      [[ 0.  0.]\n",
      " |       [ 0.  0.]]\n",
      " |      <NDArray 2x2 @cpu(0)>\n",
      " |      >>> weight.initialize(ctx=[mx.gpu(0), mx.gpu(1)])\n",
      " |      >>> weight.data(mx.gpu(0))\n",
      " |      [[-0.00873779 -0.02834515]\n",
      " |       [ 0.05484822 -0.06206018]]\n",
      " |      <NDArray 2x2 @gpu(0)>\n",
      " |      >>> weight.data(mx.gpu(1))\n",
      " |      [[-0.00873779 -0.02834515]\n",
      " |       [ 0.05484822 -0.06206018]]\n",
      " |      <NDArray 2x2 @gpu(1)>\n",
      " |  \n",
      " |  list_ctx(self)\n",
      " |      Returns a list of contexts this parameter is initialized on.\n",
      " |  \n",
      " |  list_data(self)\n",
      " |      Returns copies of this parameter on all contexts, in the same order\n",
      " |      as creation. For sparse parameters, use :py:meth:`Parameter.list_row_sparse_data`\n",
      " |      instead.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      list of NDArrays\n",
      " |  \n",
      " |  list_grad(self)\n",
      " |      Returns gradient buffers on all contexts, in the same order\n",
      " |      as :py:meth:`values`.\n",
      " |  \n",
      " |  list_row_sparse_data(self, row_id)\n",
      " |      Returns copies of the 'row_sparse' parameter on all contexts, in the same order\n",
      " |      as creation. The copy only retains rows whose ids occur in provided row ids.\n",
      " |      The parameter must have been initialized before.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      row_id: NDArray\n",
      " |          Row ids to retain for the 'row_sparse' parameter.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      list of NDArrays\n",
      " |  \n",
      " |  reset_ctx(self, ctx)\n",
      " |      Re-assign Parameter to other contexts.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      ctx : Context or list of Context, default ``context.current_context()``.\n",
      " |          Assign Parameter to given context. If ctx is a list of Context, a\n",
      " |          copy will be made for each context.\n",
      " |  \n",
      " |  row_sparse_data(self, row_id)\n",
      " |      Returns a copy of the 'row_sparse' parameter on the same context as row_id's.\n",
      " |      The copy only retains rows whose ids occur in provided row ids.\n",
      " |      The parameter must have been initialized on this context before.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      row_id: NDArray\n",
      " |          Row ids to retain for the 'row_sparse' parameter.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      NDArray on row_id's context\n",
      " |  \n",
      " |  set_data(self, data)\n",
      " |      Sets this parameter's value on all contexts.\n",
      " |  \n",
      " |  var(self)\n",
      " |      Returns a symbol representing this parameter.\n",
      " |  \n",
      " |  zero_grad(self)\n",
      " |      Sets gradient buffer on all contexts to 0. No action is taken if\n",
      " |      parameter is uninitialized or doesn't require gradient.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors defined here:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  grad_req\n",
      " |  \n",
      " |  shape\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(dense.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gluon",
   "language": "python",
   "name": "gluon"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}