CNN/ResNet.ipynb

{
 "cells": [
  {
   "attachments": {
    "image-2.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1YAAAJmCAYAAABfURFyAAAACXBIWXMAABYlAAAWJQFJUiTwAAAgAElEQVR4nOzdcZAU5Z3/8e8uOCqQGCOJgNGYyylHhIOc3hJzyTScmcRYQilRA2dQS03UKkhp1Xm5O6eLJTP+UtZdnUE4qriEiycqKkkuKrEumajbk+MMa4iLrCYEjMZEiGCUQ9BkDtnfH+vTPtPb09MzPTPPdPf7VWWxzM52P7tqf/b79Pd5uqdSqYwIAAAAAKBpvaYHAAAAAABxR2EFAAAAABFRWAEAAABARBRWAAAAABARhRUAAAAARERhBQAAAAARUVgBAAAAQEQUVgAAAAAQEYUVAAAAAEREYQUAAAAAEVFYAQAAAEBEFFYAAAAAEBGFFQAAAABERGEFAAAAABFRWAEAAABARBRWAAAAABARhRUAAAAARERhBQAAAAARUVgBAAAAQEQUVgAAAAAQEYUVAAAAAEREYQUAAAAAEVFYAQAAdJFCoWB6CACaQGEFtJDjOKaHAABIAPIEiJ+eSqUyYnoQQBIUCgUpl8tSKpVMDwUAEGOZTEYsyyJPgJjhjhXQQswwAgCiUG2AjuOQKUDMUFgBLaKHIQAAzSiXy+7HxWLR4EgANIrCCmgBfaExQQgAaJY+OcdEHRAvFFZAC+gzjLRvAACa4bcbIHkCxAeFFdAC3uDTCy0AAMLwK6zoggDig8IKiMgvCCmsAACNqHVnii4IID4orICIarVuEIQAgLCCJuS4awXEA8+xAiIoFAq+hZWI8AwSAEBomUwm8POVSqVDIwHQLO5YAW3CHSsAQBj6BJ1lWe7Htm27H5MpQPejsAIi8LtbpYciQQgAaEQ2m/V9nXZAoPtRWAFNqjXDqCMIAQD16HniLaxUvrB2F+h+400PAIgrfaFxPp93A0+FogpBx3FqFl4AAOgtf9u3b3c/fuGFFySfz4vIaM6QJUB3o7ACmpTNZsVxHLFtuyoI1WuqoCIIAQBBaq2lOv3008kRIEZoBQSaZNu2VCoVyWazMnv2bPd1FYKVSoVdAQEAAFKCwgqIiJlEAAAAUFgBAAAAQEQUVgAAAAAQEYUVAAAAAEREYQUAAAAAEVFYAQAAdAk2RALii8IKAAAAACKisAIAAACAiCisAAAAACAiCisAAAAAiIjCCgAAAAAiorACAADoQuVy2fQQADSAwgoAAAAAIqKwAgAAAICIKKwAAAAAICIKKwAAAACIiMIKAAAAACKisAIAAACAiCisAAAAACAiCisAAAAAiIjCCgAAoItYlmV6CACaQGEFAAAAABFRWAEAAABARBRWAAAAABARhRUAAAAARERhBQAAAAARjTc9gKRyHMf0ENBB27dvdz9+4YUX+PefIuzehU7gmpIuBw4cEJHRf+/8u08XMiXeeiqVyojpQSRNoVCQQqFgehgAOqRUKhGGaBsyBUgXMiW+aAUEgIjK5bLpIQAAEoJMiS9aAdvMtm3TQ0CHvPDCCyIicvrppxsdBzqHuwjoNDIlPV544QXyJGXIlPijsGozQhBILkIQnUamAMnEWrpkoBUQAAAAACKisAIAAACAiCisAAAAACAiCisAAAAAiIjCCgAAAAAiorACAAAAgIgorAAAAAAgIgorAAAAAIiIwgoAAAAAIqKwAgAAAICIKKwQS47jmB5CV3Ecp+7PhJ8ZAPjj+liNTAGaM970AAA/juOIZVk1P5fL5aRUKo15T9DndIVCQUREbNtuamzlclnK5bI4jlP3XM2GTyPjLxaL4jiOVCoV9/PZbNY9hvq52Lbd1PcMAHEWlCmFQkEKhQKZItWZon8NmQKEQ2GFrqNCzrIsKZVKYz7vveDryuVyQ+cRqQ5CFVj6cfSPvYFmWZaUy+W6RWAz6gVWoVDw/bzjOO7nvOPKZrNNjQUA4qpepqhrvN91XOVN2POIjM0Uby7Vy5QgZArQ3Sis0HXUhb1QKLgzhX5hkslk3I/VnZqggKx1Dn2mUt35qcWyLMlms26Y1DtPrSCvpVwuu+EcFFi2bbtj9wZhsVgUy7KqXm+k4ASAJImSKUo7MyWfz4c6h3pPI5min59MAdqPwgpdybZtyWazksvlJJfLucGjXlf0QkQkuN0j6BwqOPL5vHsukXeCLpfLuS0ajQozHsdx3AAM21pR6z164aeoECwWi76BqLd4AEDSkCntyRTVwuj3NWQK0ojCCl3Lb2bOe7Gu1VLhnYnMZrO+oaHOoY7Z6SDQw887lqCv0b9v/ZcA9bEKO+/3XGtBsl+LBwAkCZlS+2v8MkW1AIqMzRT1s9HzR0emIK0orNDV9IWyImPDTefXCx+mBaIdF381G6naSfxmPVXbhUh177v+3kwmM+aXAe+MqveYOvVLgx6y+hhFpKkZUwCIo6Rkih89U/SCypsp3jtYzWSKyNhClUwBKKzQhYJ2Vwpq2/AWD+rujHfGrtZuRmF2WqrXK1/r+1F9637fkz6+eu8VqS7CCoWCO9NYq+UjzC8CAJBUacsUvfgiU4DOorBC11FbzoqMDcJabRthNnzw8oZt0KydEjS7WWsm0bbtqmPXCjfVduFdJFyLen+pVHJnVsvlsrt+QP2c9M95v55WDQBJZypT6m1cIdJdmaJvvKHWiAVlit/52HodaUdhha5TKpXGtMkptUJID8N6F3bLsqRSqYw5R62eeXX8KKFRKpUkl8uNeRaIrpE2Cn0WUu08pb5W7XylvicV7H4Bz4wjgKRrJlP0tUVKrZ3wgjKl1jU2TpmiNvzwZkqYvAXShsIKXUlvS1Chob+mtj0XeSeg1Foipd7W637nqPVeFVBRQiSfz7sLi/0eQinSeADWam0RGS2a1M9AbaXr/TlRWAFIg1ZkihI2U4LyIk6ZogrTbDZb9RxJ1RpJpgDvoLBC11IXeBUY+gVftSso6uKeyWTcgAjznA01A9eJljg1qylSvci4WCxKqVQKtXuT4g3AfD5fFWh6K4r6BUL1ztP+ByCNkpwpanML9XErMkX/fr2ZUiqVJJPJkCmAB4UVupo+A6ioi30ul3OLiVoP7A37wMVO0gNQZDTA1U5NYcbinVEMeh6K/vwUVVzZtl016wgAaUGmjFUrU/zWeJEpQDAKK8SC36YS+jOZVBtfNpt1WxbCPnxRPcNDhYtfz33Q1rz6ot5651GLmUulkrv4WC0UrrcQ2Tte79i8D2qsNWuqtwQCQBolJVNyuZx7B0llSqVScddfiTSfKUFbrevIFOAdFFboairQ9HYHkdp942oGrdauRV76Nrn6ayLVM2+1AsNxnFBtH6r1T2/d0IMsaHG1V63dC72vedeM6e2AItUzjwCQBmqHu3Zlir5eSWl3pqixkymAeRRW6Fr6duL6a34Liv0u9kGbUShBuzzVm5nUZzeD3qPuUgU9R0SNW+Sd54jUCin9OH6/FKhi0W8RsfrZiNCyASBd1DVbv7a2OlMU7/U3Lpni91B6lSl+X0+mANUorNC1vD3b+rbh3vAJehp9kLCtElGECUBFD8J6IazaL7zvUz83v/PVWjcAAEnXqUxppABrRjszRWTsnaeg9VNkClCNwgpdSd/G1UvtuiTi/6BC/anxQTpRVKmZPz3I1Zj92kPUeMJsWVurdSPo2Sj6uXkGCYC06GSmtHPL8TCZ4kWmAJ3Ta3oAQBB95ky/E1NrRtAbnn5BoXSisBKp3Vcf1BoSZvZPhWsul5NMJuMuVq41k6kHpP5cKwBIOr91QPrjKGplindjBjLlHWQKMBZ3rNCVVODpYVbvKfXeHZKCFu6GCcB6ARHmmSZ+wuwqFYb6+aitcfXdAb0zs/pCZ701hC1yAaSBet5So5mibxBBpoTLFPIEaUZhha6jLujqYq2Hm3enJX3hsdrIQc1Iqgc6NjOLGGYRcRiZTCbyMfyo78nve9NnWPVfBryzmfoMYyMPkgSAOFHXc3X9a2em1Gq5S1OmZDIZMgWpRWGFrqPuoqiA8rZw6LOG6nX1Hu8DEUulkuRyuTEzgWGKrTCFWFBbSNhjNHp8PVjVc1b08+hb4HpnXP3GVigUJJfLNb1YGwC6mXfdVKszRb/OBhUTccgUdXwyBWgOhRW6TqlUqmo7UBdr798VPcz8QkcdT1epVGrOHvoVK36CHoioisKoIVgul8fMgPoFn5f+uaCZQ7WoudkWFADodvl8vuqhu63OFPV8wrhkilc7MgVIq55KpTJiehBJo89+MWMDJJea6Q279THQDDIFSD69/ZRMiS92BQQAAACAiCisAAAAACAiCisAAAAAiIjCCgAAAAAiorACAAAAgIgorAAAAAAgIgorAAAAAIiIwgoAAAAAIqKwAgAAAICIxpseQNI5jmN6CACAhCBTgGQql8umh4AWoLBqs1wuZ3oIAICEIFMAoHvRCtgG2WzW9BAAAAlBpgBAPPRUKpUR04NIokKhYHoIADogm82KZVmmh4GEI1OAdCBT4o3CCmiRQqHABREA0BJkChA/FFZAi6i1D6VSyfBIAABxl8lkxLIsMgWIEdZYAS3gOE7VPwAANEvlCJkCxAuFFdAC+japxWLR4EgAAHFHpgDxRCsg0AKZTKbq75VKxdBIAABxp2cK7YBAfHDHCojIb7cuWjcAAM3wZgrtgEB8UFgBbUDrBgCgVcgUIB5oBQQi8rYBKrQDAgAaRaYA8cUdKyCCoId20roBAGgEmQLEG3esgAhyuVzNsGPBMQCgEbXuVomQKUAccMcKaJK+oNiyrMDPAwAQRM8LMgWIJworoEn6c0Z0tm27H7PgGAAQBpkCxB+FFdAkvRc+n8+7H2ezWRPDAQDEmJ4pV155pfvxmWeeaWI4AJpAYQU0wXEcdxbRtm35n//5H/dzP/nJT3zbOAAA8OPNlKefftr93PDwMJkCxASbVwAtsGnTJrn88stFROT++++Xiy++2PCIAABxdffdd8vVV18tIiLf+9735IILLjA8IgBhcMcKAAAAACKisAIAAACAiCisAAAAACAiCisAAAAAiIjCCgAApFKhUJBCoSC5XK7q9SgP49W3TQeQLhRWAAAgdXK5nGSzWbFtWxzHcQuiQqEg5XK56S3Os9nsmEINQDqMNz0AAACATlKFjyqeSqWSiLxzt0k9U6oZ6piZTEYqlUqUYQKIGe5YAQCA1FBtfvl83n1NFUOFQiFSUaUfz7Zt2gKBlOGOFQAASI1isSgiMqbVL5fLtaSoUmzblkwm434MIPkorAAAQGo4jjOmqKrVAqjWW3mpu12qSBMRd72WTt21orAC0oFWQAAAkGiFQkEymYx7B8lxHPfvjuPU3KzCtm3JZrNu+6DaKdCyLLEsS/L5vDiO41tUec8PIPkorAAAQKLZti2VSsUtfkqlklQqFXdziaCt1W3brrlDYLlcFtu2axZV3KkC0oXCCgAApIJfW5+SzWZrfk7f6MJxHMnlcg3tIBh0XgDJQWEFAABSQW/lU8IUPWqXP/046m5VI+cFkGwUVgAAIPH8iqpGeFsCKZYAeFFYAQCA1Ahq+atHbwkUeedBwwAgQmEFAABSQG2N7i2sGim0isXimJbAMHeumr1LBiBeKKwAAEBqeIsc9fd6a60KhYK7rbp+DP1ZVl60CwLpQmEFAAASz+/BwIpt24FFkHcHQO8ugfWeU+VtIQSQTBRWAAAg0VTRVK/tz1tcqaKpUChIuVyuWXyp93gVi0X3YcIAkm+86QEAAAB0Qq3CyrZtKZfLUi6XfYsg77bqflut+x3bcRweEgykCIUVAABItDB3jvL5vORyuapCqNbXhCmWCoWC2LZNYQWkCIUVAABINMdxpFQqBb5HPQQ4l8vVfW89qnUw6nEAxAtrrAAAQKLkcjl3zZPatCLMOifbtt07V81S56WoAtKHO1YAACBRstmslMvlpu4cqQIsaBfBemj/A9KJwgoAACSKvn16M0VOlF38KKqA9KKwAgAAicMW5wA6jTVWAAAAABARd6wAAAAQO2oNnVc+nxeR0W329de4i4l2444VAAAAYse2bclms+I4jvtPNpt1d4FUn1OvAe1GYQUAAIBYsm27qmjS72CVy2Ue0oyOohUQAAAAsZXP591dIB3HcZ8lls1mKarQUdyxAgAAQGxZllVVQKnCiqIKnUZhBQAA0EV27tzpfrxjxw6DI4mPoJZAoFMorAAAAAxRrWu5XE4ymYxkMhm57bbb3M/btu2+nsvlpFAouG1vqJbNZt2P9ZZAoFMorAAAADrMcRzJ5XINFUveIozCoVqhUBjTEkgRik5i8woAAIAOcRxHisWi7y/80+ZYMnX2PBERGX/cRDn44s9lwvtPExGRvdsHZM9Q9dfohVXa1xPlcjl3B8Byuez+fIvFIluto2MorAAAADqgUCiMucs0bY4lZ1/V7xZU9Wy7s3/0z/9Y6R5TSWtxVSgUqnYA9NslMK0/G3QWhRUAAECbeYuqRgsq5eyr+ke//qPzZM9TA2MKrDQVEPrdv6DvO40/G5hBYQUAANBGuVyuqvVvwdcfb7ig8po6e557DL24KpfLUiqVIh07LlSLXz6fF5HRQku9lpafAboLhRUAAECb6BsoNHuXKsjZV/XLtI/Ok4dvnC8i6Wt981s/xZoqmMKugAAAAG3gbf+78PaBlhZVytTZ82TB1x+veV4AnUFhBQAA0GLe4kYvfNrBr7hiq3GgsyisAAAAWqxcLrsfn33lirbcqfKaOnuenH3lCvfvxWKx7ecE8A4KKwAAgBbS7xadfeUKdye/Tjj7qn6ZNmd0jZHjONy1AjqIwgoAAKCF9BbAThZVfufkrhXQORRWAAAALaLfIdLb8hq17c5+92HAjZo6e17L71qxIQZQH9utAwAAtIi+tioK9WyqZu94nX1Vv+x5ewv2ZqnnYumFWVq2cQeaQWEFAADQIlWbVhhoA1T0zTKKxWLoZzupu1K17k7pD+EFUI3CCgAAoEX0TStMmzbHkj1D4VoBg1r91F0q7lYBwSisAAAAUsiv1a/W+/Q/gSDqjmY+n0/d3U0KKwAAgBZodpOIbXf2y97tA76f23zTvKq/n31Vf+hnYk2dPU/2DDnu2PRfcnO5HFuxoy3Uf1fqv7k0FVgUVgAAAAbt3T7gFkBe3tenPjXQkocNl0olERn95bdYLAYWWeqX4gMHDsh73vOeyOdG8unFleM4Ytt2KlpJKawAAABabNpH54V+74W3D4x57d/m94iIyJceH2nRiPxZliWWZQUWWOrOwz//8z+n5s4DotPX7ak/k15c8RwrAACAFtvz1IDpITTEsiwplUpSqVSkVCqNKaAcx6F9EA2xbdu9MyoyWlwl/b8fCisAAICEa+ROU1CRxR0rNMKyrKq7VMVi0eBo2o9WQAAAgBbQi45am1F0UivGoLcKturhx0gXVVhxxwoAAAChqeKq1mYUYU2bY8m0OdHuDqkxtOIuk/fOA9CsJG/bT2EFAADQItls1v04yh2jC28f8N3UIqxtd/b7jgkwIS3/DVJYAQAAtIj+C6Re3JjEnSaYpt81TXJLKYUVAABAi6g1SSKjrXgm1lptu7Nftv3HSnc8ADqDwgoAAKCF8vm8+7Hpu1ZpacECugGFFQAAQAuZvGul362ybZs2QKCDKKwAAABaTL9r9fCN8ztSXO3dPuAWVQA6j8IKAACgxbzbk3eiJfDhG+e7H3O3Cug8CisAAIA20IubPUOO/Nv8nrbcudq7fUD+bX6P73kBdA6FFQAAQJvYtl21M9/DN85v6d2rbXf2c6cK6BIUVgAAAG1UKpWq2wL/Y2Xk4mrv9gHZfNO8qjVVFFWAWeNNDwAAACDpVMFTKBRE5O3i6j9WytlXrhARkbOv6g91nL3bB2Tbnf2yZ8gZc3yKKsAsCisAAIAO0AsfvcBSf6oiS5n20Xmy56kBERktqLzFlMjoJhn5fJ4HAQNdgMIKAACgQ/yKK8W7VXrQ1ukUVED3YY0VAABAh9m2LZVKxV1/FaZAsixLSqWS+w9FFdBduGMFAABgiGVZYwokx3HGvAdA9+OOFQAAQBexLEvK5bL7MYB4oLACAADoMoVCQYrFoulhAGgAhRUAAEAXUZtaOI4zpi0QQPeisAIAAOhS3LUC4oPCCgAAoIvo27BzxwqIDworAACALuF9tpUIxRUQFxRWAAAAXYx2QCAeKKwAAAC6BHesgPiisAIAAOgCfkWVQnEFdD8KKwAAgC4QVFjRDgh0PworAAAAw/SiyrKsMZ/nmVZA96OwAgAA6CLZbNb92LZt92PuWgHdjcIKAADAMP2O1amnnup+/OEPf9i9g8UdK6C7UVgBAAAYpu5M2bYtw8PD7uvPPvusiIy2B/q1CALoHuNNDwAAACDtbNt2i6u7777bff3jH/84LYBATHDHCrFUKBSkUChILperap+IsriXhcEAEJ3jOO712XuNLhQKTV9ng3bMA4BuwB0rxE4ul5N8Pi+WZUkmk3FfV6GrL/RthGVZksvl3I8BAI0rFouSz+fFtm3fa3Sz11fbtquu/wDQbSisECvewqdUKkm5XJZCoSDlcllKpVKk45dKJYorAGiS3zVaRFp6jc5kMlIqlbhGA+g6FFaIDdWqpwezCtZcLhc5sJVsNiu5XE4qlUpLjgcAaVDrGq1aA1t1TbVtW4rFIoUVgK7DGivERq3Fu7lcTmzbblnIqgXE9PMDQHj1rtGtYtu2ZLNZrtEAug53rBA7egGlFkHrD1NUr/uFfD6fFxGRcrks5XLZ/Vq/0C8UCi39ZQAAksxxnDFbgjd7jdY/V2tNlelrtGp71Kk88X5/rAsD0oHCCl1NDycV0CrM8vm8+zlvYFmW5a690l9T71MbX+jb2+rUHSvTwQ0A3cxbQDiOM+Ya7ff8Jb2NW3/N7xrtV5CoQs3kNVqtydV3OVRtkJZluT8XiiogPWgFRFezLEtKpZI7i2nbtpRKJXfhspoh9eMtmvTt1FUY1wtkWk0AoDbvNVpdn1WBEbS1umVZTV+j1XVfdR6Yor5vRWWG+pNNNoB0obBCLKiZP72dJMyzULyhrJ6pUi6XuRMFAC3S7ANsvXekisViQ4/OMP3sQW9xqJ7TVSgUWrahEoD4oBUQsdLMzJ93I4qwW/6qO2IAgHCauUbn83n3WqvuWrVyV9bnn39ebrvttoa/7oorrpCPf/zjdd9n27aUy+WqdnUm7oB0orBCLPi1/IVtAVEBp4orNZsYNviC2g0BAMHXaO/GFV5qbZU+kRXmuqu+Jsx777777sDP+7niiitCv1cvDgGkF62A6Hq1wqpeWOu8RVQja6coqgCgtnrX6DCTYN61SmFaC9V5612jP/ShD8mhQ4ca/ifM3SrFu0GHagkEkC4UVogNb/A2UvDkcrkx769XXBGKAFCfKpy81+hGFIvFMRtZxGnzoEKhINlsdsx6MQDpQmGFrlcvnOoVQCrwSqWS7yLjINytAoBgqrCqtaV6Peoa7d3IIi7XaH1DJL24jFtxCCA6CivEgt9zUETq7xrl3QHQ+/5aRZsK8ygzsACQBvUeexFUHHmfF+jdWKjbr9GqeFJtj7QEAulGYYWuFxRK3o0pdCqwvc9G0UNPf5ilrtYMLADgHfWKBv1Bvl7q8ReWZVVtLuQ9vt/XdsM12i8//H4e3ocIA0gudgVEVwszK6m2uvVS7X/e17LZbN1ZTp5zBQDh1bqmep/z5P0av68L8ziMRnZ2bSfvWFWhF2VNMID4orBCV1NtIEGhZNu2O/OpB63f14QJt0YeTgkAaRb2Gp3JZNyPlSjXaNu2jV+ja42VIgpIL1oB0XW8LRNhwjOfz7ekl91xnNAPEAaAtHMcJ9T1slQqteQa7V03CwDdhMIKXSWXy7k96yqAwwSoZVlSqVSkWCw2HdwUVQAQLJfLuXef1KYVYe7QWJYlpVIp0jVadRNwjQbQrSis0FX0hc7FYrHhAI0auMyCAkBt+jW60YkoVVxFOTfXaADdjDVW6Cqt6Jtvtr+dvngACMY1GgBq444VAAAAAEREYQUAAAAAEVFYAQAAAEBEFFYAAAAAEBGFFQAAAABERGEFAAAAABFRWAEAAABARBRWAAAAABARhRUAAAAARERhBQAAAAARUVgBAAAAQETjTQ+gnV588UXZtGmTvPHGG6Hef8IJJ8gXvvAFee973xv4vm9+85uyd+/eUMccP368XHjhhTJr1qzA9/3oRz+Sn/zkJzIyMhLquHPmzJEFCxYEvmfXrl3y3e9+VyqVSqhjvve975UrrrhC3vWud9V8z+HDh2XDhg2yf//+UMfMZDJy0UUXyfTp0wPf98gjj8i2bdtCHVNE5JxzzpHPfvazge959tln5eGHHw79/b/vfe+TK664QiZMmFDzPQcOHJB77rlHXn311arXn3nmGffjTZs2yfDwsIiIHHfccbJo0SL58Ic/HHju733ve7Jjx45Q4+zp6ZFzzz1XzjvvvMD3bd++XR555BE5cuRIqONOnTpVrr322sD3vPLKK3LvvffK//7v/4Y65oQJE+Syyy6TU089NfB99957rzz33HOhjtnb2yuWZcknPvGJwPc9+eST8sMf/lCOHj0a6rinnnqqXHXVVYHv2bt3r9x3331y6NChUMecNGmSLF68WKZOnRr4vn//93+Xl156KdQxw/5M0XrPP/+8fOc735E333wz1PtPPPFE+cIXviDvec97ar7nzTfflA0bNsjLL78c6pjHHHOMLFy4UD7ykY8Evu8HP/iBPPnkk6Ez5aMf/ahceOGFge/ZuXOnPPjgg/LHP/4x1DEnT54sS5culUmTJtV8z+uvvy533323vPLKK6GOeeyxx8pFF10kZ555ZuD7Nm/eLE899VSoY/b09EhfX598+tOfDnzf8PCwbN68Wf7v//4v1HFPPvlkueKKK+S4446r+Z7XXntN7rnnHnnttdeqXn/66afdjzdu3Ojm4/HHHy+XXHKJnH766YHn/va3vy0///nPQ42zt7dXPvGJT4hlWYHv+9nPfib/9V//JW+99Vao455yyily9dVXB75n3759snHjRjl48GCoYwXgj5gAACAASURBVE6cOFE+//nPyymnnBL4vrvvvluef/75UMfs7e2Vv/7rv5Zzzz038H0/+clP5NFHHw2dKaeffrosXbo08D0vvfSSPPDAAw1lyt/8zd/IySefHPi+O++8U37zm9+EOmbYnylCqlQqI0n95+qrrx4RkYb++ad/+qfAYz700EMNH/MTn/hE4DHffPPNkYkTJzZ0zJ6enpF9+/YFHnfRokUNj3XdunWBx9ywYUPDx7zgggsCj/naa6+NjBs3rqFjjh8/fuT1118PPO6nP/3phsd63333BR5zzZo1DR9z8eLFgcd86aWXGj7mxIkTR954443A486dO7fh4z7yyCOBx/x//+//NXzM6667LvCYzzzzTMPHfP/731/3//8ZM2Y0fNz//u//Djzm3//93zd8zL/7u78LPOaWLVsaPuYNN9xg5Jqa9n8uv/zyhv9drVq1KvCYmzZtaviY5513XuAxX3/99ZFjjz22oWP29PSMvPbaa4HHXbBgQcNjvfPOOwOP+c1vfrPhY1500UWBx9y/f/9IT09PQ8c89thj62aKZVkNj/U///M/A4/5L//yLw0f88orrww85vPPP9/wMd/znveM/PGPfww87kc/+tGGj/ujH/0o8JgrVqxo+Jhf/vKXA4/5s5/9rOFjnnrqqXX//z/llFMaPu6TTz4ZeMybbrqp4WPm8/nAYw4MDDR8zJtuuqkj11B1PsuyOnI+E/8kuhXwhRdeaPhrgmaWREZni1p9zHHjxkkmk2nomGFmISdOnNjQMUXqj7Udx+zt7ZVjjz224WPW+xkE3XkKOm6njzl+fOM3jo8//vi6XxeX77+ZY4b5mnb8t9qO7z9oNr+Ww4cPN/w1iO7FF19s+Gva8d9Uvevlscce2/A1dWRkpO5MfLf8/x8mU8LeqdOPWe9n1o5/V+34/hv9fUKNo6enJ/A9cfmdoplrar1jiozegW71cbvlvykypXV63q4gE+n888+Xxx57TN71rnfJd7/73brvf+9731u3ZU9ktMVuz549ocaQyWRkzpw5dQuyV199VYaHh0OHwYwZM+T9739/3fcNDg6Gblt53/veV7e9RETk5z//uezbty/UMY877jiZPXt23YvL/v375dlnnw11TBGRj3zkI/K+970v8D1/+MMfZGhoKHTbyvvf/36ZMWNG3fcNDw/L73//+6rXHMeRYrEoIiIrVqxw29SOP/546evrq3vM3/3ud7Jz585Q4+zp6ZGZM2fWbVl94403ZGhoKHTbyrRp0+SMM86o+77t27fLgQMHQh1z4sSJcs4559R930svvSS7d+8Odcze3l758z//cznhhBMC33fo0CEZGhoK3bZy2mmnyYc+9KG67/vZz34mr7/+eqhjvutd75K/+Iu/qPu+559/PtQv7W+88Yb88pe/lEWLFtEKaMB5550nP/7xj+Wkk06S++67r+77TzrpJJk5c2bd9+3cuVN+97vfhRrDscceK7Nnz66bKb///e/lmWeeaXmmbN26Vf7whz+EOmbYTHn22WdDt5cfd9xxMnfu3Lrv27dvX+hWuJ6eHjnrrLPkpJNOCnzfm2++KUNDQ6Hby6dMmVK3DV5EZMeOHWPay3/0ox/JbbfdJiIihUJBPvaxj4nI6C/Nf/mXf1n3mHv27JFdu3aFGmdvb6/MmjUrsGVVZPSX76GhodDt5R/4wAfqtsGLiAwNDYVuL580aZKcffbZdd/3m9/8Rn71q1+FOua4ceNk9uzZgcsgREbbVrdv3x46U/7kT/4k1HV627ZtoVsBTzjhBJkzZ07d94XNlMOHD8uuXbvkkksu6UgroCr6LcuSUqnU9vOZkIrC6sQTTwzdvw40Y9OmTXL55ZeLiMj9998vF198seERAWi1+fPny5YtW2TKlClN3b0Cwrr77rvd9Unf+9735IILLjA8IiC6NBRWiW4FBAAAAIBOoLACAAAAgIgorAAAAAAgIgorAAAAAIgo0YXVuHHjRKS57awBANCpLFHZAgCALtGF1YIFC+Td7363XHrppaaHAgCIuYULF8oJJ5wgl112memhAAC6UKILq+uvv15eeeUV+frXv256KAAQ2bZt22TVqlXym9/8xvRQUmn58uWyf/9+9/lCABBng4ODsmrVKnnppZdMDyUx6JEDgJj44he/KMPDw/KrX/1KVq1aZXo4AIAYu/rqq+WXv/yl7NmzhwmjFkn0HSsASJIXXnhBRESee+45swMBAMSeypRdu3aZHUiCUFgBAAAAQEQUVgAAAAAQEYUVAAAAAEREYQUAAAAAEVFYAQAAAEBEid5ufd26dXLLLbfI0qVL5fbbb+/ouXO5nDiO09Fzojt8/vOfNz0EdFipVBLLstp+nt5e5sJMuuOOO+SrX/2qXHPNNR3dmrhQKEi5XCZTUuqiiy4yPQR0WKcypVKpiIhIT09P28+VFolO6QcffFAOHjwoDzzwQEfPWygUCEAgRcrlsukhoAMefvhhOXjwoGzatKnj5yZTgPQgU+Ir0Xesjh49KiIiR44cMTaGTsw4oDvs3r1bRET+9E//1PBI0Cmd/mVXXdNghsoSMgXt9vrrr8tLL70kPT098oEPfEAmTpxoekjogE5nSiaTkUqlIiMjIx09b5IlurDqBqVSyfQQALRJJpMxPQSkDJkCJJPjONyZToBEtwICQJKoWesJEyYYHkk6qXUIrEcAkAQqUyZNmmR4JMlBYQUAMXHzzTfLqaeeKtddd53poQAAYu7mm2+W0047Tb74xS+aHkpiUFgBQEx8+ctflueee07OO+8800NJJbUOgfUIAJLgb//2b2X37t3yyU9+0vRQEoPCCgAAAAAiorACAAAAgIgorAAAAAAgIgorAAAAAIiIwgoAAAAAIkp0YTVjxgwRETnrrLMMjwQAEHdkCgAgSKILq9tvv10GBwdl8+bNpocCAIi5tWvXypNPPinf/e53TQ8FANCFEl1YiYjMmTNHJkyYYHoYABDZDTfcIJMnT5b169ebHkpqzZ49W44//njTwwCAyK699lqZPHmy3HPPPaaHkhjjTQ8AaLVcLieO40ilUun4uR3HcT8ul8tVH2ezWbFtu+NjQnJ85zvfkYMHD8pDDz0k11xzjenhAKmQy+VERKRUKnX83GQK2mnTpk3y5ptvyoMPPiiXX3656eEkAoUVUsFxnKpQCpLNZsWyrDGvFwqFqmPogRdEHatcLovjOL7HVsdTAR6FiYISnXHkyBEREXnrrbcMjwRItzhkSqFQkEKhEOqYQciU5FJZorIF0VFYITXCBoxt275Blc1m3RDMZrOSzWbHHN+yLMnn8yIiNcOuHsuyxhw7DBWyAID2KpfLZAqAMSiskAqWZYlt22OCSqeCslYAWZYVGGwqZJsNPyWfzzd1jEKhQAgCQAeoNjwyBYCOwgqJ4Hfx11/TC6tabRnFYtF9r/c4YVv0HMeRTCZT932lUilyWAIAWq9WMeHNFIVMAaBQWCH2avWR68Gl94gH9cXXCqYwC4TVGMK8NygAy+Vy6N5979cBAKIpFot1J+sqlUrVWqdayBQgXRJdWDmOI6tWrZLPfe5z7HaSYN6dkYLCyLKswNlIv3Cq166hn1fdGYuiFYuNkUy9vYl/QkZXe/TRR+Vf//VfZfHixXLZZZeZHg7aJJ/PV7XvqWu7X0sfmYI4U5POPT09hkeSHIkurNatWyebN2+WX/7ylxRWCaaHlB5wfmGUzWYDZyL9+uT199Rr36jXthGmXYOWDqA7rV27VjZv3iy//vWvKawSjEwB0KxEF1avvvqqiIjs37/f8EjQKaqnXWR0lq7WTJ/3c/rX1VNr5jLoThm7K6EVjh49anoIqUampI/KBsdxmsqUMAUNmQJTMpmMVCoVGRkZMT2UxEh0YYV0cRynKmj8QkltYOH3tWHbM2rtsBTUthFmdyVvP7vjOA0VfCJj2yIBAM3RM8WyrKYyJYx2ZYrfmMgUoL0orJAY5XK5qn1DDzy/IFQ7Oan3BLVseM8TtKjXL2RZBAwA8aJniojUzRT1WrdlCm2AQOdQWCERVJtGqVRyZ+S8Iaf+rkJQhab6M2z4BAVamG1661HjaGRMSIcZM2bIk08+KWeddZbpoQCJ1kimqDY+VVypO03dkikKmQKvGTNmyPbt22XmzJmmh5IYqSis2O0k+YrFohsaequDdybR+7paeFwqlUKfq1bbRiaTEcuyfI9Va0t4HXe1UM8Pf/hD2bVrl8yZM8f0UFJJ7crI7ozJ10imeJ+TKBL+bpV6L5kCEx5//HHZvXs3mdJCqSisWJSXbKrXvFYfuP4MK69cLtfwLF69sIqyoFiNI+zDI2upFdSIt4kTJxKABqnNQ9hEJNnIlLHIlGSaNGkSmdJiqSiskGzNPOvDtu2md1UKmiX0bqDRiHqLnfVF1ACA9oiaKX47/NU7Xy1kChAvFFaIPdu2mwoyFSphQ8uyrMCZyqC2jbDU9+E9hj6D6g37RnafAgAEi5opQduy68gUIHkorBB7ane/sPTedMuy3Ic0BoVXIzOG9d5ba1vdILWeZ6LG7heOAIDGkSlkCtAsCivEXiMBqHZ6EnnnafQqFINmGcvlct2gUsevF4JBgVXreSW1Pqcvmq71HgBAeHHLFHVeP2QK0FmJLqxOPvlkERGZOnWq4ZGgG+izinoQ6Vvolstl31nGMLN3Udo2agWwHs5BxZge0gQh0B5TpkwRETIFo8Jkiv53HZkCJFOiC6t8Pi+9vb1y7bXXmh4KDNMD0LbtMbN7+sLjei0c7RhbrdfVIupsNuu+T99ByjuTqT/4GEBr9ff3y4QJE+RLX/qS6aHAsLCZEjRh186xiciYdWJkCtB+iX4YxxlnnCHf+ta35K/+6q9MDwUGeWcVa82+qc85jiOZTKbq6zOZTN1/RMT92nr/6MWUCjXvuNR7VLHn99wS27alVCpVfV/6M1eQLOvXr5dFixbxfBpDpk+fLuvXr5e5c+eaHgoM0guUsJmib3eey+Xamin6ei/vuNUxyRSIiKxbt04WLVokTzzxhOmhJEai71gBIqOzdmEX4vq9R319q8ek5PN53x2o9BnOMLOF6j380p1cK1askH379skxxxzT8K5lAFqjFZnS6v9/9eOVSiXfHGg2U5Bctm3LgQMHZNKkSXLuueeaHk4iUFghcbwtF40+rNEbhI1+faNqHb+Zc9ILn2xvvPGGiIgcPnzY8EiA9Gh1prT7Ok2mICyVKYcOHTI8kuRIdCsgAAAAAHQChRUAAAAARERhBQAAAAARUVgBAAAAQESJL6yGhobcxXkAEGe9vYm/ZHe9oaEhefPNN00PAwAiq1QqIiLS09NjeCTJkeiUvvHGG6Wvr08WLFhgeigAENnRo0dNDyHVbrjhBunr65PPfe5zpocCAJGp56WNjIwYHklyJLqw+sUvfiEiIsPDw4ZHAgCIOzIFABAk0YUVAAAAAHQChRUAAAAARJSKwopFeQCSYMqUKVV/orPU5iFsIgIgCVSWTJs2zfBIkmO86QEknVoYCABR3XXXXbJlyxZZtGiR6aHAEDIFQKvce++98sQTT8gll1xieiiJkYrCqtO7nWSz2Y6eD0A6nHPOOXLOOeeYHkZqqV0ZO707I5kCoB36+vqkr6/P9DASJRWFVadZliW2bUu5XDY9FAAdYNu26SEgwcgUIF3IlPiisGoT/qdIn0KhINlsVizLMj0UAAlDpqQPmQLEDytwgRYqFoumhwAASAgyBYgX7lgBLVIoFEwPAQCQEGQKED+JvmM1btw4EREZP576Ee2lB6DjOAZHAqBdVJaobAHahUwB4inRhdWCBQvk3e9+t1x66aWmh4KE00OQ1g0gmRYuXCgnnHCCXHbZZaaHgoQjU4B46qlUKp3dixxIGMdxJJfLVb1WKpVYcIyW++lPfypbtmyRiy++WE477TTTwwHQBoVCYUwbIJmCdhgcHHSfY3XKKae0/XzqOXyWZUmpVGr7+UygRw6IyG8L5GKxSAii5ZYuXSrPPfecDA8Pyze+8Q3TwwHQIWQK2mHx4sXy29/+Vnbt2iVr1qwxPZxESHQrINAJfguM6YlHO7z88ssiIrJ3717DIwHQLmQKOmXfvn0iIrJnzx7DI0kOCisggqBdmwhCAEAjyBQg3iisgDZhwTEAoFXIFKD7UVgBEajZRb/ed2YXAQCNIFOAeKOwAppUq2XDtm33Y4IQABAGmQLEH4UV0CQ9BPP5vPtxNpt1P6Z1AwAQBpkCxF+iC6t169bJ5MmT5aabbjI9FCSMPmto27Zs27bN/fvQ0FBVGwczjGiV3t5EX7K73h133CGTJ0+Wr3zlK6aHgoTxZor+jB/HccgUtEWlUhERkZ6eHsMjSY5Ep/SDDz4oBw8elAceeMD0UJAwlmVJpVIR27Ylm83Kqaee6n7utNNOk3w+L6VSiYc6Agny8MMPy8GDB2XTpk2mh4KE0TPFtm35sz/7M/dz55xzDpkCxESiHxB89OhRERE5cuSI4ZEgqVTvu/cXLYIP7aCuaTBDZQmZgnbR11PpyBS0QyaTkUqlIiMjI6aHkhiJvmMFAAAAAJ1AYQUAMTFx4kQREZkwYYLhkaSTWofAegQASaAyZdKkSYZHkhyJbgVUCEEASdDf3y/f//73ZdmyZaaHkkoUVgCS5Ktf/ar84Ac/kBtuuMH0UBIjFYUVvaMAkuCaa66Ra665xvQwUkutcWOtG4AkuO666+S6664zPYxEoRUQAAAAACKisAIAAACAiCisAAAAACAiCisAAAAAiIjCCgAAAAAiSvSugDNmzJDHHntMzjrrrI6fO5fLdfycMGffvn3uxytXrpS1a9caHA06KZvNim3bpoeBDpgxY4Zs2bKFTEHb/e53v3M/zufzcvvttxscDTqJTIm3nkqlkui9yIeGhuTMM8/s6AM1C4WCFAqFjp0PgFmlUkksy2r7eQ4fPiy7du2SOXPmtP1c8Ld9+3Y588wz5fjjj+/YOckUIF06lSmHDh2S3bt3dyxTMpmMiIhYliWlUqkj5+y0xLcCzpkzp6NFFYD0KZfLHTnPwoULpa+vT/r7+ztyPow1e/bsjhZVANKnU5ny2c9+Vvr6+uRrX/taR86XBoluBewGD+1M9A1BINUWTu/p6PmefvppERF56qmnOnpedA8yBUimHYMDcsvS+R09J5nSeom/YwUASfHWW29V/QkAQLNUlhw5csTwSJKDwgoAAAAAIqKwAgAAAICIKKwAAAAAICIKKwAAAACIKNGFleM4smjRIrnnnntMDwUAIuvtTfQlu+s9+uijsmjRInnggQdMDwUAIqtUKiIi0tPT2R1ukyzRKb1u3TrZvHkz+/MDACJbu3atbN68WW677TbTQwEAdKFEF1avvvqqiIjs37/f8EgAILqjR4+aHkKqkSkAkiSTyYiIyMgIz8drlUQXVgAAAADQCakorOgdBZAE48ePFxGRcePGGR5JOqk1bqx1A5AEKlPUn4iOdACAmDj//PNFROSCCy4wPBIAQNx95jOfEREypZUorNCVdgwOBH7ulqXzAz8X9PWtsmNwoCPnCTq/snFNf82fidKpnwva56677pJdu3bJddddZ3ooQKxEzZRO6LZM2bimP/C9ZEr83X///bJr1y656qqrTA8lMVJx749FefGycU2/bFy9Um7d8LjM6ps35vP3rV7pBpDf53cMDshiWRF4DhUItc7hd0wRkeHBARne6lSFyZLlK2TJsv6aX3ff6pV1j+9n8fIVNcemAk993jsm9YvArRsed8exY3BAZg5aob5fdK8PfvCDpoeQWmrzEDYRiZdWZEqtzylxz5SF03tkifb54a3O6CeWjf5xy9L5MnOuVTUuMiUZyJTWSkVhhXiZ2TdPZvU5csvS+W7A7BgckGHPzNiw9pq62KvAqXehnznXcmfc9BBTBUm9WTh1/KCgapY6d73A2vj29+oNYPVLgCqqdDMJQAApo2eKKny6MVNmzrXeHmvwuRoVJlOWLF9RN1MWL39nwlL9nMgUoBqFFbrOrL55MmvDPLll6Xz3Qj+zb55sXL3SDYVZffPcGbUdgwNVQRAmlJYs65cly/rdmUz12sy5lgxvdaqOoV5rZDbS+72Eoc9EBs1YqrHWcsvS+aPn1cbZ7AwnAMSdnimq8CFTxo6/Fr9Mce9oAahCYYWudeuGx8f0eHvvEG1c0181E6jaNfxmB/3CyxsmS5b1u60Pulu2zq95jKhU+Kmxhw3aWjObfneqlFprBeqFLgDE3a0bHh9zDQzKFL0FMO2ZUuvryRSgGoUVupq6MKtQu2/1ShmeO+B+Xp81qwrDpe+8RyR4lq8dF3/VZhJ07DDhpwpL/Tj6jKjXwunVjxbQ215EqkNcf412DgBpoK87FQnOFCVpmeItoMgUoHUSXVidfPLJIiIydepUwyNBqwTtmqRaHlRwDg8OyMbVK922D+9x2rngVoWbXhh6zzf89vfindnTFwm7rSrajOfMvnmyZPnbx/BZ9Oylh51+N8u7wQWAYFOmTBERMiVJ4popG9f0jymyamWKPkE3Ov7qgqhWpqh1X17q837ZJUKmIN0SXVjl83np7e2Va6+91vRQ0ICgi7N3Bk7NtOktG9XhtnLM7JzfAmN1rFp94ypIgrbdrRUmaozewkaZ6fl+dgwO+IaZon+Pt2yd73684+3dpbytLWrM3mO2+xcBIGn6+/tlwoQJ8qUvfcn0UNCApGaKiP/dMb8OB79JN8WbKUuWr6gqsGpliheZAiS8sDrjjDPkW9/6lulhoEEz51qycfVKdwcn3ZhdnN4OLTWzGFSQKLP65rk7IA1vdarOUW/npmae2eHOMNb4nhQ9AMO0kqgi7KGdI1XFk1poPGbtwOqVY44b5ueF7rF+/Xr5/ve/LzfeeKNks1nTw0md6dOny/r1600PAw0ylSnejgI/UTPFm2G6RjNF3zJeTdr5ZYq7VfxWZ8z6MTIlXtatWyc/+MEP5Oabb5Zzzz3X9HASIdGFFeJpybLRHnC1g5MeGqOzdGN7wdWFXi8ehrV2Br9zqPcvnN4jt2543N3VyY8KnId2NvdMNP18fi0cjQSg/sBKb6Cqn919q1e63z8PcEyOFStWyL59++SYY46hsAJCMpUp9TYS6pZM0e+AhckUxbtxlAjbr8eNbdty4MABmTRpEoVVi/SaHgDgR80A6n9/aOeIG0L63x/aOVLVZhe2kFiyrL+jveBLlvW7vfk73u7VFxkN65lvf79hZhVVuNXaqUn9LJYs6x+zRkD12hOC8fTGG2+IiMjhw4cNjwSIl3qZsmT5ipZkSlDLXavVyhS1QYXKgTCCdg8MkymIJ5Uphw4dMjyS5KCwQteqVfgsWb5iTNDpgTCstSnU6/dW4dqpvvAly/plWLvjJPJ2O0ed3Z68x9DHrB7c6DfzOXOu5fbPVz2DhLtYAFImKFO8u+I1myne63O7BWVK2DGon4s3U3zXBNfIFJ6VCIyiFRCx4N3y1fuaO6P2dkAuWdbf9ELaoNnJsM8yqUXfyEL1/c96+0GVIs1t06sCzW/r3yXL+mVH3+iY1fmWLOt3f0FgoTGANEpypuiFYqcyRY2bTEHaUVihq6kgU+0VeoAEUe0Ji0O0ZSyc3lPVhhe0S5Pf58Kui3K3y337/foYZw6OBtTG1SvrPsyx1jNHvL8oeFs71Fa7atMLtsQFkDZpyJSq9VcdyBR90wsg7Sis0LX8trBVOzbVCh23eFAzbnVmz2r1h6sdkIIMaz3t9c6hLwyutfBZX1wdFKz6M0fcWUnPWGuNy6/lBQDSwC9T6t3ZIVOqXx9zHjIFqJL4wmpoaEjOPPNMmTBhgumhoEHDPhss1Hr2hz5Tpi70YVoSggq1cC0NwYGiFhT7bX/ud75bNzwutyydLxtXr6y5sYRqtxj9WYzd9SloZnX0mCtp2Yip3l6WxZo2NDQk06dPl+OPP970UNAgv0xRWpUpQYVaKzJFb/3rhkzxHgPxUqlURESkp2dsayyak+jC6sYbb5S1a9fKJz/5SXn00UdNDwcN8q4D0p8GrwT1rodp7VBtFO1Ua3cpVdTp34/aTCNML/99b4erHoB6Ief39WzBHm9Hjx41PYRUu+GGG2T9+vXyqU99Sh555BHTw0GDamWKLujaG7ZdsJ2Zou4uqeu+X6bo2p0p+vsQP5lMRiqVioyMNLftP8ZKdGH1i1/8QkREhoeHDY8EjVJPvddnDdWCWn2Wzq8/3d1yNsLi3VaZ1TdvzAJgfcy1AjjszJ/+TCt9Mwy/Xnc9IHcMDvg++wRAbWRKfAVliv6aN1OqtjEPmSntfIyFX4FDpgDdI9GFFeLLu3WrCsV6s2b6g3NV+0OtC72JwssbUFFaJ27d8HjVAmb32VQ1ZlX1wlRtliFitvAEgE5wd7rT7laFvYsjEj5TOt0SR6YA3YXCCl3JW0TpF3C/94lUt/apXZ/UjkXe2bYwLRv1Hnzo13bR8DEitk8Ma+HnzhquXinDW50xd/b0n437MyMIAaSAt4hqV6YEtQt2e6bsGBxoOFPU5hlkCjAqFYUVi/LiRV3UVeB5w01d/JWZc62qBb3qgq7v+uQNwnoXf322Lop27Ja0cc3oc6j08VXtcvX2z2LH0oHq55l4FiSr51kRhPExZcoU2b17t0yZMsX0UFJJbR7CJiLx0mimLF6+gkwJmSl6sUqmxM+UKVPkxRdflGnTppkeSmKkorBCvAxrs2WKHm76lrT68zv8+sBrbqFbZ3FxrZ5ynd6LXstDO6MtCPV7iKU6t/oevN/jkmX9VQ+zVA+M9PtZqPaW4a2OyLJIQ0UH3HXXXbJlyxZZtGiR6aEAsdFopszqmyfDb18/w2RKmE2QyBR0o3vvvVeeeOIJueSSS0wPJTF6KpVKYrcCOf/88+Wxxx6TE088UV5++eWOnbdQKEihUBCR6BdBdJ7f7oONvreRYwTZuGb0WSRsY9udy11o8AAAH41JREFU1C8ptm2LbduGR4N2mz9/vmzZssWd5e0UMiXeyBSEoRfWSc2UTCYjIiKWZUmpVDI8mvbgjhXg0Ujg1Hpvq0KLVgoAiDcyBUgPGsUBAAAAICIKKwAAAACIiMIKAAAAACJKdGE1btw4EREZP56lZACAaFSWqGwBAECX6MJqwYIF8u53v1suvfRS00MBAMTcwoUL5YQTTpDLLrvM9FAAAF0o0YXV9ddfL6+88op8/etfNz0UAIjspz/9qaxataqjW33jHcuXL5f9+/fLbbfdZnooABDZ4OCgrFq1Sl566SXTQ0kMeuQAICaWLl0qzz33nAwPD8s3vvEN08MBAMTY4sWL5be//a3s2rVL1qxZY3o4iZDoO1YAkCTqQed79+41PBIAQNzt27dPRET27NljeCTJQWEFAAAAABHRCthmtyydb3oIAICEIFMAoHtRWLXZjsEB00MAACQEmQIA3YtWwDbIZrNiWZbpYQDoAMuyxLZt08NAgmWzWdNDANAhZEq8cceqDSzLorBKoUKhQFENoOUsy5JKpWJ6GOgwMgWIn0TfsVq3bp1MnjxZbrrpJtNDQUoUi0XTQ0CC9fYm+pLd9e644w6ZPHmyfOUrXzE9FKQEmYJ2UhM2PT09hkeSHIlO6QcffFAOHjwoDzzwgOmhIAUKhYI4jmN6GADa5OGHH5aDBw/Kpk2bTA8FKUCmAPGT6MLq6NGjIiJy5MgRwyNB0hUKBfdjghDtoq5pMENlCZmCdiNT0AmZTEZEREZGRgyPJDkSXVgBJtC6AQCIolwuux+TKUB8UFgBLeCdXWSGEe0wceJEERGZMGGC4ZGkk1qHwHoEtJueIeQJ2kVlyqRJkwyPJDlSUVgRgmgnvagC2qm/v18uvPBCWbZsmemhpBKFFTrBL1MortAOX/3qV+XCCy+UG264wfRQEqOnUqkktrHy/PPPl8cee0xOPPFEefnll00PBwmlepR1lmVJqVQyMBoA7TJ//nzZsmWLTJkyRV588UXTw0FCkSlIKvXfdpL/e07FHSugXWrdraIdEADQKDIFiDcKK6BNWHAMAGgVMgXofhRWQARB66uYXQQANIJMAeKNwgpokh6AlmX5fkwQAgDCqJUpOjIF6G4UVkAL5PN59+NsNut+TOsGACAM/dlVeqbYtu1+TKYA3S3RhdWMGTNEROSss84yPBIkkZpdtG1bdu7c6b4+depUd7aRBcdAcpApaCeVFbZty49//GP39UwmU5UpALrXeNMDaKfbb79drrzySjnzzDNNDwUJVKlU3OLqxBNPdF8/6aST3NnGfD5fs6UDaNThw4dl165dMmfOHNNDSaW1a9fKddddR6agLfRMOf30093XZ82aJR/72Mckm81WdUQAUR06dEh2795NprRQogsrEeE/FrSVatHYtGlT1euWZVFQoeUWLlwoP/7xj+Uf//Efpb+/3/RwUmn27Nmmh4AEU5ly9913V71OpqAdPvvZz8rWrVtl5cqV8g//8A+mh5MIiW4FBIAkefrpp0VE5KmnnjI8EgBA3JEprUdhBQAx8dZbb1X9CQBAs1SWHDlyxPBIkoPCCgAAAAAiorACAAAAgIgorAAAAAAgIgorAAAAAIgo0YWV4ziyaNEiueeee0wPBQAi6+1N9CW76z366KOyaNEieeCBB0wPBQAiq1QqIiLS09NjeCTJkeiUXrdunWzevFm+9rWvmR4KACDm1q5dK5s3b5bbbrvN9FAAAF0o0YXVq6++KiIi+/fvNzwSAIju6NGjpoeQamQKgCTJZDIiIjIyMmJ4JMkx3vQAkspxHCmXy6aHgQ555pln3I83bdokw8PDBkeDTrNt2/QQkHBkSrqoB7eKiGzcuFG2bdtmcDToNDIlvlJRWHW6d7RQKEihUOjoOdE9vv3tb5seAjosm82KZVltP8/48aOX7HHjxrX9XBhLrXHr9Fo3MiXd7r//ftNDQId1MlMqlYqbLYgu0a2AANAJnbqTcP7554uIyAUXXNCR8wEAOq9TmfKZz3xGRMiUVqJEbbNbNzxuegjokB1bR/9dz5o73/BI0Cm3LO3sv+u77rpLCoWCfPCDH+zoedE9yJT02PX0oIwbP17+5CN/YXoo6IDhwQHZuHplR895//33y69//WsypYVSUViZXJQ3q2+esXOjs/h3jU4gAM1Rm4eY3ESE60x68O86jTpbWImQKa1GKyAAAAAARERhBQAAAAARUVgBAAAAQEQUVgAAAAAQUaILq5NPPllERKZOnWp4JACAuJsyZYqIkCkAAH+J3hUwn89Lb2+vXHvttaaHAgCIuf7+fpkwYYJ86UtfMj0UAEAXSvQdqzPOOEO+9a1vyV/91V+ZHgoARLZ+/XpZtGhRxx4eiWrTp0+X9evXy9y5c00PBQAiW7dunSxatEieeOIJ00NJjETfsQKAJFmxYoXs27dPjjnmGMlms6aHAwCIMdu25cCBAzJp0iQ599xzTQ8nERJ9xwoAkuSNN94QEZHDhw8bHgkAIO5Uphw6dMjwSJKDwgoAAAAAIqKwQuJsXNMvG9f0mx5GlR2DA7JjcMD0MAAADdq4pr/rrt9kCtCdWGOFxBne6siOwQFZsqy/pcf1C7Fhz2vDW53Ar3lo50jgOVpRELb6+waANBve6sjwVkdmbZjX0uO2O1N2DA6MOV4zyBQgPAorpMKOwQG5Zen8UO9dsnyFb5Dct3plqBnCWX3zRERk5lxLZs61Rj9++7V6Y9y4emWoMQYhBAGgvTau6Q99vTaVKcNkCtBxiS+shoaG5Mwzz5QJEyaYHgoMm9U3T3YMDrgh5Sco5G7d8Lj7eb9jLJzeI7P65smtGx6PNM4ly1eECk2vsCGN+OrtpXvbtKGhIZk+fbocf/zxpocCw8JmSq3rOZkC0yqVioiI9PT0GB5JciS6sLrxxhtl7dq18slPflIeffRR08OBQbP65oksF9mxdEBmzrV8Z+B2DA6Mft4ngLwtekHtFWHa+Wb2zasZxkGfCzI8l577pDt69KjpIaTaDTfcIOvXr5dPfepT8sgjj5geDgxasqxfdvQNhMoUP2QKukEmk5FKpSIjI8HLFBBeogurX/ziFyIiMjw8bHgkaKcdgwNyn9buoIJAb/27dcPjdYNFHcP7vkZa9MIuKL61xb36ANqPTEmHjWv6q9Y2kSkAwkp0YYV0GK4RPLXCaHirI7Js7Ou1Wjpm9c2ru+mESOvaNsKuBQMAtJ7aAMmLTAFQD4UVYm/Jsv6qNoyF00d7hf2Ca8nyFb4zhSow1cLgWsLMHAa9J0w7xqy+eXXHAQBoD72QURsf1SpwyBQAulQUVizKSw89gIIWFdf6XNAC3zA7Cwb11IuEax9ZvHxFU/3wSL4pU6bI7t27ZcqUKaaHkkpq8xA2EUkPtfZJteQ1em0mU9DNpkyZIi+++KJMmzbN9FASIxWFFdJDX2t1y9L5Y+5ajYbcShn2BGStXng/tYIsqG0jzNa8fouXm1k4TIAm11133SVbtmyRRYsWmR4KkAr6Wqv7Vq8c8ywrlSkb11R3TnRDpvghU6C799575YknnpBLLrnE9FASIxWFFbudpIM+o6j+vGXp/KpQUgGxcfVKNwSbnYlst0aevaXM6pvX8odYonucc845cs4555geRmqpXRnZnTEd/LKBTEGS9PX1SV9fn+lhJEoqCiukw32rV1YF2a0bHpeF03vGBKHqiVczjGpmcfHyFaHOExRMOwYH3DVezVLfQysWLQMAmkOmAGgUhRUSYeOaftkxOCC3bni8qh1QBZ4+e7hk2WgLxcbVK2Wmdncr7OxirYctqgXOfmHa7IMWu23GEwDSoJlMGd7qjD7bqksyRW9jVMgUoL0orBB76pkgKsjuEy0El/X7PhxRhaOaKQw7syhS/2GLfp8L86BFv+emNIvwBIDmRMkUtdFEN2SK/rVkCtAZFFaIvXptF36BoGYY1ecbCY37Vq+sClpdMz3sOjWOqM8dCbNTFABgrKRkin5XjUwBOiPRhdW4ceNERGT8+ER/m6m3Y3BAljSxnaza5KKR53ssCZiFrPfckplzrcAx6iHo1wevgjFMjzwBCLSeyhKVLUimZjNF6ZZM0b+WTAE6I9EVx4IFC2Tr1q1y6aWXmh4K2qiZmbRbls53Cxk1y6hvleun3iykah2pdxw/3gD1nmfjmtFjesN+x+CA3Ld6Jc8pATpg4cKFMjQ0JJdddpnpoaCNms0UEem6TNHPVXVsMgVoi0QXVtdff71cf/31poeBNmu2qFqyfIUsWdYvC6f31A3CjWv6fRcCe4Vp2/ALLPUMK78FzOp5JWq8OtX/r3apIgiT7ac//als2bJFLr74YjnttNNMDyd1li9fLsuXLzc9DLRZqzIlaO1UpzLFL9PIFCiDg4Puc6xOOeUU08NJBB4fj1RROz3ps4CqDWJjwC5LfgVPs/weBKwvlK71uVpFn1oHoO9chWRaunSp3HzzzVIoFEwPBYAEZ4oquPy0MlN8x+XZKt7vc2QKFi9eLDfffLN87WtfMz2UxEj0HStAp7f/eR/wqO8SWGsWr95DEhdO72nqOSGqJcPbR69aMkSqe+D1oFZFmlov5n2+CpLl5ZdfFhGRvXv3Gh4JADIFcbdv3z4REdmzZ4/hkSQHhRUST2+l8As4kXfaJdTzrfTXdgwO+N5lqkWFWhDf7Xo949LbP4JmPkXe6dXfMTjgPqQSANB6qpVOhEwBUI3CConn9prXCEBFPZ/klqXzR3vfl73z9RtDtkTsGAz3bJEly6v7+P12hvJ7TopIdQuJ9z23LJ1ft7cfABBds5nSyAPjyRQgXiiskDiLl6+QmYPvtECo4Asz4+bXdrFkWXtn65Ys6/cNzmbaL27d8HjVtu0AgGgWL19RdYep1kOC/ahM0d/b7ta6VmbK4uUrZLGwQyAQFoUVEsdvE4hGCiMTAdLKcxKAANA6fpnSyHU2zplCngCNYVdAAAAAAIgo0YXVunXrZPLkyXLTTTeZHgoARNbbm+hLdte74447ZPLkyfKVr3zF9FAAILJKpSIiIj09PYZHkhyJTukHH3xQDh48KA888IDpoQAAYu7hhx+WgwcPyqZNm0wPBQDQhRJdWB09elRERI4cOWJ4JAAQnbqmwQyVJWQKgCTIZDIiIjIyMmJ4JMmR6MIKAID/3979x0Z513EA/5RCk7UEGBthC1mKUxiIZjjjcHPeRU0zRxamnWZOdJOh/DBziXGaTHsZ84g/4h/+scU4k2W64JwGJ8mYohXGdVsm6jbQatZsQ9ABbiQIEwZ2DvzD9CwI99zxtDzc09crIfSu3376OY7r5973fO8pAJwJghVAk+jo6IiIiPb29ow7GZuG3ofg/QhAHgzNlIkTJ2bcSX6MidOtG4JAHqxatSoeffTRuPXWW7NuZUwSrIA8+epXvxq//OUvY+XKlVm3khtjIljZOwrkwdKlS2Pp0qVZtzFmDb3HzXvdgDxYvnx5LF++POs2csVWQAAAgJTGxBGrLP3onlVZtwBATpgpkE/9WypZt8AIEKxG2Y/uvivrFgDICTMF4OxlK+AoKBQKWbcAQE6YKQDNwRGrUVAsFmNwcDAqFYd1x5IHHnggbrrppqzbIAPFYjHrFsgxM2Xs2rZtW1x66aVZt8EZZJ40t1wHq7lz58amTZti3rx5mXx/D46xpa+vLyLc75BXc+fOjSeffNJM4Ywol8tRKBTc79BEWgYHB3N9LvKtW7fG7Nmz/UJNRl1bW1sUi8Xo7e3NuhVy6tChQ/H888/H/Pnzs25lzNq2bVvMnj07zjnnnKxbIefMFEbbwYMH44UXXjhjM6WtrS0iItf/r3P/Hqv58+cLVYy6crkcEWGrDqNq0aJFcfnll8eqVauybmXMuvTSS4UqRt3wmWKuMFquueaauPzyy+PrX/961q3kRu6DFZwJQ0MwQrhi9PzhD3+IiIhnn302406AM2X16tVZt0BOmSkjT7CClIaHqghDkNHzxhtvHPc3kE9erONMGJol//73vzPuJD8EKxhhtm4AcLpOfLEuQriCZiFYQUonG4IAcDqGzjA7nJ0Q0BwEK0jhVKHKEATgdJzs6JSdENAcBCtI4WSvLEYYggA0zg4IaG65DlaVSiW6u7vjhz/8YdatkENJ4clRK0bauHG5/pF91tu4cWN0d3fHT37yk6xbIadqBSszhZE2ODgYEREtLS0Zd5IfuZ7S9957b6xfv975+RkVpzpaVSwWI8KbjSFvvvOd78T69evjm9/8ZtatkEPDQ9XQHBnOTgg4++U6WO3bty8iIvbu3ZtxJ+RR0hCMEK4YWUePHs26hTHNTOFMKRQK1Y9LpVL1Y0etGEltbW0REXHs2LGMO8mPXAcrGC3DQ1WpVIr9+/dXL/f09FQ/NgQBqMfQXCkWi9HR0VG9fsqUKXZCQJMYE8HK3lFGWqlUqr6KeGKwKhaL1T/DQxakNX78+IiIaG1tzbiTsWnoPW7e68ZoKJVKUSwWo1AoxN///vfq9eedd14UCoUoFovR29ubYYfkzdBMGfqb9PxLwmkaHq5mzpwZO3furH7O8GM0fPCDH4yHHnooFi5cmHUrwAgbvuVvzZo11Y+nTJly3OdgpFx99dXxs5/9zEwZQYIVQJN44IEHolwuR2dnZ9atANDkfvzjH8fOnTvNlBE0JvYzeFMekBcGYHaGTh7iJCJAXpgpI2tMBCsAAIDRJFgBAACkJFgBAACkJFgBAACklOtgNX369IiIuPDCCzPuBIBmd8EFF0SEmQLAyeX6dOs9PT0xbty4+PSnP511KwA0uVWrVkV7e3ssW7Ys61YAOAvl+ojVrFmz4v7774/3vOc9WbcCkNp9990X3d3d0dfXl3UrY9Ill1wS9913XyxYsCDrVgBSu/fee6O7uzueeuqprFvJjVwfsQLIkzvvvDNeeeWVmDBhQhQKhazbAaCJlUql2L9/f0ycODGuuOKKrNvJhVwfsQLIk9deey0iIg4dOpRxJwA0u6GZcvDgwYw7yQ/BCgAAIKUxFayuuuqqaGtrq/vPe9/73sSaX/va1xqqOWPGjOjv769Z8ze/+U1ccMEFddecNGlS3H333Ym9XnbZZQ31eu211ybW/PKXv9xQzc7OzvjLX/5Ss+amTZti+vTpddecPHly3H///TVrHj58OObNm9dQrzfeeGPi7b/tttuira0tKpVK9bpaNWfOnBkvv/xyzZqPPPJInH/++XX3ee6558batWtr1ty/f3/Mnj27odu/fPnyxNu/ZMmShmrOmTMn/vnPf9as+eCDD8a5555bd83zzz8/fvGLX9SsuWvXrpg5c2ZDvX7hC19IvP3XX399QzXf/va3J9b83ve+F5MnTz7p13tV8eyyYMGChu7/D3zgA4k177rrroZqXnTRRTEwMFCz5hNPPNHwTPnud7+b2Os73vGOhnrt7u5OrPnFL36xoZqdnZ3x0ksv1ay5YcOGmDZtWt01p0yZEg8++GDNmgcPHow5c+Y01OunPvWpxNu/YsWKaGtri1tuuaV63Yc+9KFT1nzLW94S+/btq1lz7dq1MXXq1Lr7nDp1aqxbt65mzVdeeSUuvvjihm7/rbfemnj7P/7xjzdUc+7cuXH48OGaNX/wgx/ElClT6q45bdq06O3trVlzx44d0dnZ2VCvd9xxR+LtX7RoUUM1L7vsssSa99xzT0yaNOmkXz84OBgRES0tLYl1qM+YCVb79u2L3/72tw19zZYtWxKfBCc9+E60d+/exDcJPvXUU4k/KIc7cuRI/OpXv6q5Zvv27YmB7kRJNSMi8Qntifbs2RO/+93vaq554okn4h//+EfdNQ8fPpx4PwwMDMTzzz9fd82IiJ/+9KeJax599NGGau7evTueffbZmmsee+yxePXVV+uueejQodi4cWPNNVu3bo0dO3bUXTPivwEvSdLwPdH27dvjj3/8Y801mzZtamir26uvvhqbN2+uueaZZ56J3bt3110zor77tp5/o+EGBgYSH4e//vWvE58otLe3N/R9GXm7du1KfCyf6PHHH098bG/YsKGhmi+//HJs2bKl5ponn3xyxGfKc889F3/605/qrhkRsX79+sQ1P//5zxuquWfPnvj9739fc01fX18cOHCg7pqvvfZa4kzp7++P7du3110zIuLhhx9OXNPo7f/rX/8aW7durblm8+bNDb0oc/DgwXjsscdqrtm6dWtioD1RPfd/0ouEJ3rxxRfjz3/+c801GzdurG55q8eBAwfi8ccfr7nm6aefjj179tRdM6K++7bRx39/f3/ic5ve3t44cuRIzTUdHR0NfV9ObcycvGLq1Klx++23NxSu3vWud1V/F9ap3HbbbdHe3h7/+te/6qo5bdq0uOaaa2quWbRoUUPh6pxzzonPfvazNddcfPHFsWLFisQfQENaWlqiWCwmrrv99ttjzZo18frrr9dV98ILL4z3ve99Nddcf/310d/fX/ft7+joiM985jM118yfPz9uueWWeOGFF+qqOW7cuLj66qsT191xxx2xdu3aeOaZZ2L//v0RETVPKnDRRRfFlVdeWbPm4sWLY8eOHXWHq4kTJ8bNN99cc82CBQvi5ptvTjxaOKS1tTWuu+66xHXlcjkeeeSReOONN+qq+6Y3vSnx9i9ZsiT27t1bd7iaNGlS4tHFq666Km688cbYtWtXXTXHjx8fH/vYxxLXlcvl2LhxYxw9erSuurNnz463ve1tNdcsW7Ysjhw5csonAh0dHXW98svomjFjRnzuc5+Lbdu21bW+paUlrrzyypg0aVLNdZ///Ofj+9//fvWV5CTTp0+Prq6umms+/OEPx9NPP133z9T29vZYsWJFzTVz5syJZcuWxXPPPVdXzXHjxsX73//+xHVf+tKX4qGHHqp7psyYMSPxRC433HBDDAwM1B2uOjo6YunSpTXXvPvd744lS5bEiy++WFfN1tbWxNkfEfGVr3wlHn744di9e3f1SOS8efPivPPOO+n6zs7OxLNUfvKTn4yXXnqp7nA1ceLE+MQnPlFzzRVXXBGLFy+Ov/3tb3XVbG1tjY985COJ677xjW/Ehg0b6v6Z+uY3vzne+c531lyzdOnSOHDgQN0zZfLkyfHRj3605ppisRg33HBD3eFqwoQJsXjx4sR1d955Z2zevDmOHTtWV923vvWtMWvWrJprVq5cGUePHj3lC3YdHR2xcuXKur4fyVoGBwfru/eAU+rq6qpuB6z3CREAnMyaNWuq2wHXrVsXCxcuzLgjSK+trS0i/htMG93x1SzGzFZAOBPqOcoHAED+CFYAAAApCVYAAAApCVYAAAApCVYAAAApCVYAAAApCVYAAAApCVYAAAApCVYAAAApCVYAAAApCVYAAMCoqVQq1Y8LhUKGnYwuwQpGwPAfGAAAnJxgBQAAcBpWr16ddQtnhGAFAACMikqlUt3ZUywWo1gsZtzR6BGsAACAEVcul6Orq6t6uaenJ8NuRp9gBQAAjKhyuRzlcrl6uVQq5fpoVUTE+KwbAADg5Hp6euLb3/521m1A3U52Qq9SqRSlUimDbs4swQoA4CwyMDBQ/bi/vz/DTiC9sRKqIgQrAICzyiWXXJJ1C3DaisVi9ZTqYyVQDRGsAADOUuvWrYuFCxdm3QZQByevAAAASEmwAgAASEmwAgAASEmwAgAASEmwAgAASEmwAgAASEmwAgAASEmwAgAASEmwAgAASEmwAgAASEmwAgAASEmwAgAASEmwghFUKBSybgEAgAwIVgAAACkJVgAAACkJVgAAACkJVgAAACkJVgAAACkJVgAAACkJVjCCnG4dgLTGjfvf07PXX389w06ARozPugHIg97e3ujr64tisZh1KwA0uVmzZsW3vvWtmDBhQlx33XVZtwPUqWVwcPBY1k0AAAA0M1sBAQAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhKsAAAAUhqfdQPQbCqVSvT19f3f9YVCISLiuM8VCoUoFotnrDcAmo+5AvnQMjg4eCzrJqDZdHV1RaVSqV4ulUpRKpWO+9zw6wCgFnMFmp+tgHAaenp6jrt84iuNhh8AjTBXoPk5YgWnqVwuR7lcrl7u7e2tDkLDD4BGmSvQ3ByxgtNUKpWO2+fe1dVVvR4AGmWuQHMTrCCFpK0bANAIcwWal2AFKRSLxeNeXaxUKse9+RgAGmGuQPPyHitIqa2tLYrF4nGDb3BwMMOOAGhWlUolurq6zBVoQo5YQQpdXV1RKpX+b+vG0L54AGjE6tWro7e311yBJtRaKpVWZd0ENKNyuRydnZ1RKpVi5syZEfG/vfA7d+6MQqFQvR4AkgzNlZtuuslcgSbkiBU0qFKp/N8pcSMiCoXCcZdXr15tXzwAiU41V05krsDZbXzWDUCzOdnvFKlUKtHX1+eUuAA07FRz5cTrgLObk1cAAACkZCsgAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASoIVAABASv8BtwVmcmTp78IAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "让我们聚焦于神经网络局部：如图 图所示，假设我们的原始输入为x，而希望学出的理想映射为f(x)（作为图上方激活函数的输入）。 左图虚线框中的部分需要直接拟合出该映射f(x)-x，而右图虚线框中的部分则需要拟合出残差映射f(x)。 残差映射在现实中往往更容易优化。 以本节开头提到的恒等映射作为我们希望学出的理想映射f(x)，我们只需将右图虚线框内上方的加权运算（如仿射）的权重和偏置参数设成0，那么f(x)即为恒等映射。 实际中，当理想映射f(x)极接近于恒等映射时，残差映射也易于捕捉恒等映射的细微波动。 右图是ResNet的基础架构–残差块（residual block）。 在残差块中，输入可通过跨层数据线路更快地向前传播。\n",
    "\n",
    "ResNet沿用了VGG完整的3x3卷积层设计。 残差块里首先有2个有相同输出通道数的3x3卷积层。 每个卷积层后接一个批量规范化层和ReLU激活函数。 然后我们通过跨层数据通路，跳过这2个卷积运算，将输入直接加在最后的ReLU激活函数前。 这样的设计要求2个卷积层的输出与输入形状一样，从而使它们可以相加。 如果想改变通道数，就需要引入一个额外的1x1卷积层来将输入变换成需要的形状后再做相加运算。\n",
    "\n",
    "![image-2.png](attachment:image-2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from torch.nn import functional as F\n",
    "from tools import tool_pytorch_017 as ty\n",
    "\n",
    "class Residual(nn.Module):  #@save\n",
    "    def __init__(self, input_channels, num_channels,\n",
    "                 use_1x1conv=False, strides=1):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(input_channels, num_channels,\n",
    "                               kernel_size=3, padding=1, stride=strides)\n",
    "        self.conv2 = nn.Conv2d(num_channels, num_channels,\n",
    "                               kernel_size=3, padding=1)\n",
    "        if use_1x1conv:\n",
    "            self.conv3 = nn.Conv2d(input_channels, num_channels,\n",
    "                                   kernel_size=1, stride=strides)\n",
    "        else:\n",
    "            self.conv3 = None\n",
    "        self.bn1 = nn.BatchNorm2d(num_channels)\n",
    "        self.bn2 = nn.BatchNorm2d(num_channels)\n",
    "\n",
    "    def forward(self, X):\n",
    "        Y = F.relu(self.bn1(self.conv1(X)))\n",
    "        Y = self.bn2(self.conv2(Y))\n",
    "        if self.conv3:\n",
    "            X = self.conv3(X)\n",
    "        Y += X\n",
    "        return F.relu(Y)\n",
    "\n",
    "# 查看输入和输出形状一致的情况。\n",
    "blk = Residual(3,3)\n",
    "X = torch.rand(4, 3, 6, 6)\n",
    "Y = blk(X)\n",
    "Y.shape\n",
    "\n",
    "# 在增加输出通道数的同时，减半输出的高和宽。\n",
    "blk = Residual(3,6, use_1x1conv=True, strides=2)\n",
    "blk(X).shape\n",
    "\n",
    "# ResNet的前两层跟之前介绍的GoogLeNet中的一样： 在输出通道数为64、步幅为2的7x7卷积层后，接步幅为2的3x3的最大汇聚层。 \n",
    "# 不同之处在于ResNet每个卷积层后增加了批量规范化层。\n",
    "b1 = nn.Sequential(nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3),\n",
    "                   nn.BatchNorm2d(64), nn.ReLU(),\n",
    "                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))\n",
    "\n",
    "# GoogLeNet在后面接了4个由Inception块组成的模块。 ResNet则使用4个由残差块组成的模块，每个模块使用若干个同样输出通道数的残差块。 \n",
    "# 第一个模块的通道数同输入通道数一致。 由于之前已经使用了步幅为2的最大汇聚层，所以无须减小高和宽。 \n",
    "# 之后的每个模块在第一个残差块里将上一个模块的通道数翻倍，并将高和宽减半。\n",
    "def resnet_block(input_channels, num_channels, num_residuals,\n",
    "                 first_block=False):\n",
    "    blk = []\n",
    "    for i in range(num_residuals):\n",
    "        if i == 0 and not first_block:\n",
    "            blk.append(Residual(input_channels, num_channels,\n",
    "                                use_1x1conv=True, strides=2))\n",
    "        else:\n",
    "            blk.append(Residual(num_channels, num_channels))\n",
    "    return blk\n",
    "\n",
    "# 接着在ResNet加入所有残差块，这里每个模块使用2个残差块。\n",
    "b2 = nn.Sequential(*resnet_block(64, 64, 2, first_block=True))\n",
    "b3 = nn.Sequential(*resnet_block(64, 128, 2))\n",
    "b4 = nn.Sequential(*resnet_block(128, 256, 2))\n",
    "b5 = nn.Sequential(*resnet_block(256, 512, 2))\n",
    "\n",
    "# 最后，与GoogLeNet一样，在ResNet中加入全局平均汇聚层，以及全连接层输出。\n",
    "net = nn.Sequential(b1, b2, b3, b4, b5,\n",
    "                    nn.AdaptiveAvgPool2d((1,1)),\n",
    "                    nn.Flatten(), nn.Linear(512, 10))\n",
    "\n",
    "# 每个模块有4个卷积层（不包括恒等映射的1x1卷积层）。 加上第一个7x7卷积层和最后一个全连接层，共有18层。 \n",
    "# 因此，这种模型通常被称为ResNet-18。 通过配置不同的通道数和模块里的残差块数可以得到不同的ResNet模型，例如更深的含152层的ResNet-152。 \n",
    "# 虽然ResNet的主体架构跟GoogLeNet类似，但ResNet架构更简单，修改也更方便。这些因素都导致了ResNet迅速被广泛使用。\n",
    "\n",
    "# 查看ResNet的分层结构输出\n",
    "X = torch.rand(size=(1, 1, 224, 224))\n",
    "for layer in net:\n",
    "    X = layer(X)\n",
    "    print(layer.__class__.__name__,'output shape:\\t', X.shape)\n",
    "    \n",
    "batch_size = 128\n",
    "train_iter, test_iter = ty.load_data_fashion_mnist(batch_size, resize=224)\n",
    "\n",
    "def evaluate_accuracy_gpu(net, data_iter, device=None): #@save\n",
    "    \"\"\"使用GPU计算模型在数据集上的精度\"\"\"\n",
    "    if isinstance(net, nn.Module):\n",
    "        net.eval()  # 设置为评估模式\n",
    "        if not device:\n",
    "            device = next(iter(net.parameters())).device\n",
    "    # 正确预测的数量，总预测的数量\n",
    "    metric = ty.Accumulator(2)\n",
    "    with torch.no_grad():\n",
    "        for X, y in data_iter:\n",
    "            if isinstance(X, list):\n",
    "                # BERT微调所需的（之后将介绍）\n",
    "                X = [x.to(device) for x in X]\n",
    "            else:\n",
    "                X = X.to(device)\n",
    "            y = y.to(device)\n",
    "            metric.add(ty.accuracy(net(X), y), y.numel())\n",
    "    return metric[0] / metric[1]\n",
    "\n",
    "\n",
    "#@save\n",
    "def train_ResNet(net, train_iter, test_iter, num_epochs, lr, device):\n",
    "    \"\"\"训练AlexNet模型\"\"\"\n",
    "    def init_weights(m):\n",
    "        if type(m) == nn.Linear or type(m) == nn.Conv2d:\n",
    "            nn.init.xavier_uniform_(m.weight)\n",
    "    net.apply(init_weights)\n",
    "    print('training on', device)\n",
    "    net.to(device)\n",
    "    optimizer = torch.optim.SGD(net.parameters(), lr=lr)\n",
    "    loss = nn.CrossEntropyLoss()\n",
    "    animator = ty.Animator(xlabel='epoch', xlim=[1, num_epochs],\n",
    "                            legend=['train loss', 'train acc', 'test acc'])\n",
    "    timer, num_batches = ty.Timer(), len(train_iter)\n",
    "    for epoch in range(num_epochs):\n",
    "        # 训练损失之和，训练准确率之和，样本数\n",
    "        metric = ty.Accumulator(3)\n",
    "        net.train()\n",
    "        for i, (X, y) in enumerate(train_iter):\n",
    "            timer.start()\n",
    "            optimizer.zero_grad()\n",
    "            X, y = X.to(device), y.to(device)\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat, y)\n",
    "            l.backward()\n",
    "            optimizer.step()\n",
    "            with torch.no_grad():\n",
    "                metric.add(l * X.shape[0], ty.accuracy(y_hat, y), X.shape[0])\n",
    "            timer.stop()\n",
    "            train_l = metric[0] / metric[2]\n",
    "            train_acc = metric[1] / metric[2]\n",
    "            if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:\n",
    "                animator.add(epoch + (i + 1) / num_batches,\n",
    "                             (train_l, train_acc, None))\n",
    "        test_acc = evaluate_accuracy_gpu(net, test_iter)\n",
    "        animator.add(epoch + 1, (None, None, test_acc))\n",
    "    print(f'loss {train_l:.3f}, train acc {train_acc:.3f}, '\n",
    "          f'test acc {test_acc:.3f}')\n",
    "    print(f'{metric[2] * num_epochs / timer.sum():.1f} examples/sec '\n",
    "          f'on {str(device)}')\n",
    "    \n",
    "lr, num_epochs, batch_size = 0.05, 10, 256\n",
    "train_iter, test_iter = ty.load_data_fashion_mnist(batch_size, resize=96)\n",
    "train_ResNet(net, train_iter, test_iter, num_epochs, lr, ty.try_gpu())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3.8",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}