3.12 权重衰减(线性回归)的从零实现及简洁实现.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 以下代码来自：https://trickygo.github.io/Dive-into-DL-TensorFlow2.0/#/chapter03_DL-basics/3.12_weight-decay?id=_3123-%e4%bb%8e%e9%9b%b6%e5%bc%80%e5%a7%8b%e5%ae%9e%e7%8e%b0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建数据\n",
    "\n",
    "%matplotlib inline\n",
    "# import d2lzh as d2l\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, models, initializers, optimizers\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "n_train, n_test, num_inputs = 20, 100, 200\n",
    "true_w, true_b = tf.ones((num_inputs, 1)) * 0.01, 0.05\n",
    "\n",
    "features = tf.random.normal(shape=(n_train + n_test, num_inputs))\n",
    "labels = tf.keras.backend.dot(features, true_w) + true_b\n",
    "labels += tf.random.normal(mean=0.01, shape=labels.shape)\n",
    "train_features, test_features = features[:n_train, :], features[n_train:, :]\n",
    "train_labels, test_labels = labels[:n_train], labels[n_train:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化参数\n",
    "def init_params():\n",
    "    w = tf.Variable(tf.random.normal(mean=1, shape=(num_inputs, 1)))\n",
    "    b = tf.Variable(tf.zeros(shape=(1,)))\n",
    "    return [w, b]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义L2惩罚项\n",
    "def l2_penalty(w):\n",
    "    return tf.reduce_sum((w**2)) / 2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义线性回归模型\n",
    "def linreg(X, w, b):\n",
    "    return tf.matmul(X, w) + b\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义\n",
    "def squared_loss(y_hat, y):\n",
    "    return (y_hat - tf.reshape(y, y_hat.shape)) ** 2 /2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义优化器\n",
    "def sgd(params, lr, batch_size, grads):\n",
    "    \"\"\"Mini-batch stochastic gradient descent.\"\"\"\n",
    "    for i, param in enumerate(params):\n",
    "        param.assign_sub(lr * grads[i] / batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义训练和测试\n",
    "batch_size, num_epochs, lr = 1, 100, 0.003\n",
    "net, loss = linreg, squared_loss\n",
    "optimizer = tf.keras.optimizers.SGD()\n",
    "train_iter = tf.data.Dataset.from_tensor_slices(\n",
    "    (train_features, train_labels)).batch(batch_size).shuffle(batch_size)\n",
    "\n",
    "def fit_and_plot(lambd):\n",
    "    w, b = init_params()\n",
    "    train_ls, test_ls = [], []\n",
    "    for _ in range(num_epochs):\n",
    "        for X, y in train_iter:\n",
    "            with tf.GradientTape(persistent=True) as tape:\n",
    "                # 添加了L2范数惩罚项\n",
    "                l = loss(net(X, w, b), y) + lambd * l2_penalty(w)\n",
    "            grads = tape.gradient(l, [w, b])\n",
    "            sgd([w, b], lr, batch_size, grads)\n",
    "        train_ls.append(tf.reduce_mean(loss(net(train_features, w, b),\n",
    "                             train_labels)).numpy())\n",
    "        test_ls.append(tf.reduce_mean(loss(net(test_features, w, b),\n",
    "                            test_labels)).numpy())\n",
    "    # 没有的需要安装一下 d2lzh和mxnet,用来画图\n",
    "#     d2l.semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss',\n",
    "#                  range(1, num_epochs + 1), test_ls, ['train', 'test'])\n",
    "    print('L2 norm of w:', tf.norm(w).numpy())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 20.85445\n"
     ]
    }
   ],
   "source": [
    "#  观察过拟合\n",
    "fit_and_plot(lambd=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 简介实现\n",
    "def fit_and_plot_tf2(wd, lr=1e-3):\n",
    "    net = models.Sequential()\n",
    "    net.add(layers.Dense(1))\n",
    "    net.build(input_shape=(1, 200))\n",
    "    w, b = net.trainable_variables\n",
    "    optimizer = optimizers.SGD(learning_rate=lr)\n",
    "    train_ls, test_ls = [], []\n",
    "    for _ in range(num_epochs):\n",
    "        for X, y in train_iter:\n",
    "            with tf.GradientTape() as tape:\n",
    "                l = loss(net(X), y) + wd * l2_penalty(w)\n",
    "            grads = tape.gradient(l, net.trainable_variables)\n",
    "            optimizer.apply_gradients(zip(grads, net.trainable_variables))\n",
    "        train_ls.append(tf.reduce_mean(loss(net(train_features),\n",
    "                             train_labels)).numpy())\n",
    "        test_ls.append(tf.reduce_mean(loss(net(test_features),\n",
    "                            test_labels)).numpy())\n",
    "#     d2l.semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss',\n",
    "#                  range(1, num_epochs + 1), test_ls, ['train', 'test'])\n",
    "    print('L2 norm of w:', tf.norm(w).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 1.4484388\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot_tf2(0, lr)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (tf2)",
   "language": "python",
   "name": "tf2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}