Section_07_Sequence_Labeling_using_RNNs.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Section 07 - Sequence_Labeling using RNNs.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "mount_file_id": "1hy0-T3oK6-nmZN9AUVyHNIdQv8DLRSba",
      "authorship_tag": "ABX9TyNgq7FndK3qTQiBa5HzmQgA",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "926c6c75c58b436693bd94d1938706fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3f6f17e1767b433ca7b3112730b43203",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_145e8369af46408db4c15f8ec896ae3d",
              "IPY_MODEL_14391856f15540ee8e72d99f1eead095"
            ]
          }
        },
        "3f6f17e1767b433ca7b3112730b43203": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "145e8369af46408db4c15f8ec896ae3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8c08cb96ce2f43209c82e2935be373a7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_17cecdc3fb7c4ec5acbc1dbf489f270f"
          }
        },
        "14391856f15540ee8e72d99f1eead095": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9afe8f957b704af7b692f2e10e264a4a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 250/? [00:13&lt;00:00, 18.15it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_59725d85f2bd4cbc83cf7b3a7f4b3c29"
          }
        },
        "8c08cb96ce2f43209c82e2935be373a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "17cecdc3fb7c4ec5acbc1dbf489f270f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9afe8f957b704af7b692f2e10e264a4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "59725d85f2bd4cbc83cf7b3a7f4b3c29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "91109a153a9447b2910ab1e6b4e546d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3cd33e45a6d9451c92e18242266214e8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_763d1ce8825f4429bb0f7fc258eba8dc",
              "IPY_MODEL_e664845f97db4e2e9e4dae87e846246b"
            ]
          }
        },
        "3cd33e45a6d9451c92e18242266214e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "763d1ce8825f4429bb0f7fc258eba8dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_174ccc3c013844058cde1c0aa3f0dace",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_002eb334b02b4b80a7c648fde57f4835"
          }
        },
        "e664845f97db4e2e9e4dae87e846246b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bcf76552de574086ab162ef8f94b2552",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 82/? [00:01&lt;00:00, 58.70it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_962812fa4cc942bb85ffac0ad8cabdd6"
          }
        },
        "174ccc3c013844058cde1c0aa3f0dace": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "002eb334b02b4b80a7c648fde57f4835": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bcf76552de574086ab162ef8f94b2552": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "962812fa4cc942bb85ffac0ad8cabdd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "53c1829b4fc84b2c9368d22c4c16f53d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1cde1b7b4f894ef9a75e0603c82778ce",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c4b8dd176a2b490ca3ea95d22808b1b1",
              "IPY_MODEL_cd21705f1f4c44b0b20e979a3f7e08aa"
            ]
          }
        },
        "1cde1b7b4f894ef9a75e0603c82778ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4b8dd176a2b490ca3ea95d22808b1b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2adcca2bc07f421d936a6fbee862a26b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_db99b541ce9340b9907f0b800ff66511"
          }
        },
        "cd21705f1f4c44b0b20e979a3f7e08aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9a9621caaa5942f7925ea758a0285799",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 250/? [00:42&lt;00:00,  5.94it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35e2c1ab8ac74f5492f31c77ce2dfe74"
          }
        },
        "2adcca2bc07f421d936a6fbee862a26b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "db99b541ce9340b9907f0b800ff66511": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a9621caaa5942f7925ea758a0285799": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "35e2c1ab8ac74f5492f31c77ce2dfe74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a6278e2d14e1446494ea283fb8511065": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_08c98dc05a3b44db9d00e4ebafee82a0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0805935de46b4ba6b62f95853e181c14",
              "IPY_MODEL_834c90ad3130486bba9253b501ad0597"
            ]
          }
        },
        "08c98dc05a3b44db9d00e4ebafee82a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0805935de46b4ba6b62f95853e181c14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a601a14da8ea4de09e50874bd34fd6b2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2696177531db45e2a96761477d214c56"
          }
        },
        "834c90ad3130486bba9253b501ad0597": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7b153df9f5b146c483d905209b6a83e4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 82/? [00:14&lt;00:00,  5.75it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b44362021c024b57bc40929e86026643"
          }
        },
        "a601a14da8ea4de09e50874bd34fd6b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2696177531db45e2a96761477d214c56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b153df9f5b146c483d905209b6a83e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b44362021c024b57bc40929e86026643": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e149ccfeb914251839a4f582d4294fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_eeb2b123540948ada2dbec590dbe3ce2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_50a38bfdffe843c785209e74c7e0e27c",
              "IPY_MODEL_1f7f967da2ab439790368cb495de86cf"
            ]
          }
        },
        "eeb2b123540948ada2dbec590dbe3ce2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "50a38bfdffe843c785209e74c7e0e27c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6e958fd6e3d3469ab3e8c02f2e390d50",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b610f15d937143c18a742fdefe7ee840"
          }
        },
        "1f7f967da2ab439790368cb495de86cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ce73330c28f04b32859b39cbbe0ddd01",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 250/? [00:14&lt;00:00, 17.84it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f0bf53e5e768466db05fd23323cd5dbd"
          }
        },
        "6e958fd6e3d3469ab3e8c02f2e390d50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b610f15d937143c18a742fdefe7ee840": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce73330c28f04b32859b39cbbe0ddd01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f0bf53e5e768466db05fd23323cd5dbd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8fb34bb1dfd2439ab5a75405f2dc6583": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_955a22169b594cb191603957f097c935",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7b7903cea18d45e09091d2d905686967",
              "IPY_MODEL_452265a65d0f436e97b155cb70faa597"
            ]
          }
        },
        "955a22169b594cb191603957f097c935": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b7903cea18d45e09091d2d905686967": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3d43ac2e21ac46cea2dcfe7ac65c4242",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3bc83167e7054a6d93f2018d6191e92d"
          }
        },
        "452265a65d0f436e97b155cb70faa597": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9b9c6ed8e82a4c9d8435ccd1abc967d3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 82/? [00:00&lt;00:00, 322.30it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_63e42c663ccc4de881f9e6bf39987ffe"
          }
        },
        "3d43ac2e21ac46cea2dcfe7ac65c4242": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3bc83167e7054a6d93f2018d6191e92d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9b9c6ed8e82a4c9d8435ccd1abc967d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "63e42c663ccc4de881f9e6bf39987ffe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "be49cf3c4ba749bba46c5a325eef1bed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_66db0538de204d07a054b795beed3aa0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8e6f8cc90f03496a8c8e6c386234d838",
              "IPY_MODEL_0317315bacdb4d349133743e17fc45c0"
            ]
          }
        },
        "66db0538de204d07a054b795beed3aa0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8e6f8cc90f03496a8c8e6c386234d838": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b9aa50bac4b3414f936f381bf3c5383b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9e75d90f73d14de6b4864b83e6d29da4"
          }
        },
        "0317315bacdb4d349133743e17fc45c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f4f33e870d624c02b94a2645da6e34d0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 250/? [00:13&lt;00:00, 17.94it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1194f583e2a940babec8c82bc3de224e"
          }
        },
        "b9aa50bac4b3414f936f381bf3c5383b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9e75d90f73d14de6b4864b83e6d29da4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f4f33e870d624c02b94a2645da6e34d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1194f583e2a940babec8c82bc3de224e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95794444b54d4dba901db75a5049f746": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_749f695dc539449c80204633b74da0b4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_21ffe1a7b70745f6a71f9319ee51dda2",
              "IPY_MODEL_8f7b5790307248b799127e25d0513c4d"
            ]
          }
        },
        "749f695dc539449c80204633b74da0b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "21ffe1a7b70745f6a71f9319ee51dda2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3d23cb3673a942ca9c1efb25795e2e6a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7199119089694dbaa0b81dd34dd296fb"
          }
        },
        "8f7b5790307248b799127e25d0513c4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6c423c9497924a6da9c5baa59c0252fb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 82/? [00:00&lt;00:00, 271.99it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6e14f3c2cd69466b80af70639c0450f1"
          }
        },
        "3d23cb3673a942ca9c1efb25795e2e6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7199119089694dbaa0b81dd34dd296fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c423c9497924a6da9c5baa59c0252fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6e14f3c2cd69466b80af70639c0450f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "df178ac82b294bba8382a4954e13f064": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7ded50dfc1ba4a3190db316bce961bdf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e8161483e9074fd18494c707cd3ce466",
              "IPY_MODEL_b359439258574552b7034d3999c4f44e"
            ]
          }
        },
        "7ded50dfc1ba4a3190db316bce961bdf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e8161483e9074fd18494c707cd3ce466": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b55718ee707247b689a4806bb30440c7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1799e75ffbd7429ca76b0ef1b22ce09e"
          }
        },
        "b359439258574552b7034d3999c4f44e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9c3728a64cbf41868f4c8e54b8c39994",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 125/? [00:00&lt;00:00, 351.97it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cadfda6c7314495d84b0e5a8366a8440"
          }
        },
        "b55718ee707247b689a4806bb30440c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1799e75ffbd7429ca76b0ef1b22ce09e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9c3728a64cbf41868f4c8e54b8c39994": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cadfda6c7314495d84b0e5a8366a8440": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rrajasek95/nlp-243-notebooks/blob/main/Section_07_Sequence_Labeling_using_RNNs.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VTer4X89DwqX"
      },
      "source": [
        "# General utility\n",
        "import os\n",
        "from collections import Counter\n",
        "from itertools import chain\n",
        "import numpy as np\n",
        "from tqdm.auto import tqdm\n",
        "\n",
        "# Torch imports\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "\n",
        "# Gensim for loading word embeddings\n",
        "import gensim\n",
        "from gensim.test.utils import datapath\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "\n",
        "# Plotting stuff\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import math\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K0aHji35AhOh"
      },
      "source": [
        "## Data download"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kitGM04f1Rui",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4552f123-3484-4f70-f3a4-8c5c9b7ca56e"
      },
      "source": [
        "!wget https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ark-tweet-nlp/twpos-data-v0.3.tgz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-11-30 22:13:07--  https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ark-tweet-nlp/twpos-data-v0.3.tgz\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.218.128, 173.194.79.128, 108.177.119.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.218.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 433577 (423K) [application/octet-stream]\n",
            "Saving to: ‘twpos-data-v0.3.tgz’\n",
            "\n",
            "twpos-data-v0.3.tgz 100%[===================>] 423.42K  --.-KB/s    in 0.002s  \n",
            "\n",
            "2020-11-30 22:13:07 (171 MB/s) - ‘twpos-data-v0.3.tgz’ saved [433577/433577]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xXpWiSYyAj3P",
        "outputId": "c06b27cb-3726-4b4f-d041-f7aafe9b36ce"
      },
      "source": [
        "!tar xvzf twpos-data-v0.3.tgz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "twpos-data-v0.3/\n",
            "twpos-data-v0.3/anno_changes/\n",
            "twpos-data-v0.3/daily547.conll\n",
            "twpos-data-v0.3/full_data/\n",
            "twpos-data-v0.3/oct27.conll\n",
            "twpos-data-v0.3/oct27.splits/\n",
            "twpos-data-v0.3/README.txt\n",
            "twpos-data-v0.3/scripts/\n",
            "twpos-data-v0.3/scripts/toconll.sh\n",
            "twpos-data-v0.3/oct27.splits/oct27.dev\n",
            "twpos-data-v0.3/oct27.splits/oct27.test\n",
            "twpos-data-v0.3/oct27.splits/oct27.train\n",
            "twpos-data-v0.3/oct27.splits/oct27.traindev\n",
            "twpos-data-v0.3/oct27.splits/splits.txt\n",
            "twpos-data-v0.3/full_data/daily547.supertsv\n",
            "twpos-data-v0.3/full_data/oct27.supertsv\n",
            "twpos-data-v0.3/anno_changes/oct27_reanno_changes_to_supertsv.diff\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1qlG004LChP5"
      },
      "source": [
        "# Create raw data splits"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lU-nC-rbAnsy"
      },
      "source": [
        "def load_split(data_path):\n",
        "    with open(data_path, 'r') as data_file:\n",
        "        examples = []\n",
        "        \n",
        "        tokens = []\n",
        "        tags = []\n",
        "\n",
        "        for line in data_file:\n",
        "            stripped_line = line.rstrip()\n",
        "\n",
        "            if len(stripped_line) == 0:\n",
        "                # End of tag sequence for the example is indicated by empty line\n",
        "                examples.append((tokens, tags))\n",
        "                \n",
        "                tokens = []\n",
        "                tags = []\n",
        "            else:\n",
        "                # In the simple CoNLL format, (token, tag) is placed in each line as\n",
        "                # <TOKEN>\\t<TAG>\\n\n",
        "                token, tag = stripped_line.split(\"\\t\")\n",
        "\n",
        "                tokens.append(token)\n",
        "                tags.append(tag)\n",
        "        \n",
        "        return examples"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GsPRSV52DZoS"
      },
      "source": [
        "splits_path = os.path.join(\"twpos-data-v0.3\", \"oct27.splits\")\n",
        "\n",
        "train_path = os.path.join(splits_path, \"oct27.train\")\n",
        "dev_path = os.path.join(splits_path, \"oct27.dev\")\n",
        "test_path = os.path.join(splits_path, \"oct27.test\")\n",
        "\n",
        "train_data = load_split(train_path)\n",
        "dev_data = load_split(dev_path)\n",
        "test_data = load_split(test_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mtj00jJhDuhF",
        "outputId": "cf1717c1-758b-48db-d6cb-dae6a9c1226b"
      },
      "source": [
        "dev_data[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['@ciaranyree',\n",
              "  'it',\n",
              "  'was',\n",
              "  'on',\n",
              "  'football',\n",
              "  'wives',\n",
              "  ',',\n",
              "  'one',\n",
              "  'of',\n",
              "  'the',\n",
              "  'players',\n",
              "  'and',\n",
              "  'his',\n",
              "  'wife',\n",
              "  'own',\n",
              "  'smash',\n",
              "  'burger'],\n",
              " ['@',\n",
              "  'O',\n",
              "  'V',\n",
              "  'P',\n",
              "  'N',\n",
              "  'N',\n",
              "  ',',\n",
              "  '$',\n",
              "  'P',\n",
              "  'D',\n",
              "  'N',\n",
              "  '&',\n",
              "  'D',\n",
              "  'N',\n",
              "  'V',\n",
              "  '^',\n",
              "  '^'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "urEZfUBiILcP"
      },
      "source": [
        "## Load Twitter Word2Vec embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1YxS0mZ2IO2K"
      },
      "source": [
        "# You can download the embeddings from here: https://github.com/FredericGodin/TwitterEmbeddings\n",
        "\n",
        "# Load embeddings into Gensim class containing matrix\n",
        "weights_path = \"/content/drive/MyDrive/NLP243/Sections/Section 07/word2vec_twitter_tokens.bin\"\n",
        "word2vec_weights = gensim.models.KeyedVectors.load_word2vec_format(datapath(weights_path), binary=True, unicode_errors='ignore', limit=1000000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ayHZB1NvShGB",
        "outputId": "3a4ca3b5-f6f0-4ccf-86de-e825a32ab20f"
      },
      "source": [
        "# Test out the embeddings\n",
        "print(word2vec_weights.most_similar('running'))\n",
        "\n",
        "# print(word2vec_weights.most_similar('runnnnning'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('runnin', 0.7631831169128418), ('Running', 0.7020593881607056), ('runing', 0.6962482333183289), ('runnning', 0.6624175310134888), ('runnig', 0.6258898377418518), ('runnng', 0.5988162159919739), ('sprinting', 0.5925309062004089), ('run', 0.585367739200592), ('runnung', 0.5655999183654785), ('running/walking', 0.5529230237007141)]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qsMcivOAH-W9"
      },
      "source": [
        "## Feature Classes\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QS_PUq2HICHQ"
      },
      "source": [
        "class W2VSequencer(object):\n",
        "    def __init__(self, gensim_w2v):\n",
        "\n",
        "        self.w2v = gensim_w2v\n",
        "        self.w2v.add(['<s>'], [np.zeros((400,))])\n",
        "        self.w2v.add(['</s>'], [np.zeros((400,))])\n",
        "        self.w2v.add(['<pad>'], [np.zeros((400,))])\n",
        "        self.w2v.add(['<unk>'], [np.zeros((400,))])\n",
        "\n",
        "        self.bos_index = self.w2v.vocab.get('<s>')\n",
        "        self.eos_index = self.w2v.vocab.get('</s>')\n",
        "        self.unk_index = self.w2v.vocab.get('<unk>')\n",
        "        self.pad_index = self.w2v.vocab.get('<pad>')\n",
        "\n",
        "    def encode(self, tokens):\n",
        "        # Input will look like:\n",
        "        # [<s>, w1, w2, ..., wn, </s>]\n",
        "        sequence = [self.bos_index.index]\n",
        "        for token in tokens:\n",
        "\n",
        "            index = self.w2v.vocab.get(token, self.unk_index).index\n",
        "            sequence.append(index)\n",
        "        sequence.append(self.eos_index.index)\n",
        "\n",
        "        return sequence\n",
        "\n",
        "    def create_padded_tensor_with_lengths(self, sequences):\n",
        "        # Given a list of sequences, pad all to the same length\n",
        "\n",
        "        lengths = [len(sequence) for sequence in sequences]\n",
        "        max_seq_len = max(lengths)\n",
        "        tensor = torch.full((len(sequences), max_seq_len), self.pad_index.index, dtype=torch.long)\n",
        "\n",
        "        for i, sequence in enumerate(sequences):\n",
        "            for j, token in enumerate(sequence):\n",
        "                tensor[i][j] = token\n",
        "        \n",
        "        return tensor, lengths\n",
        "\n",
        "class Sequencer(object):\n",
        "    def __init__(self, tokens, bos_token='<s>', eos_token='</s>', unk_token='<unk>', pad_token='<pad>'):\n",
        "        self.word2idx = {}\n",
        "        self.idx2word = {}\n",
        "\n",
        "        self.pad_index = self.add_token(pad_token)\n",
        "        self.unk_index = self.add_token(unk_token) \n",
        "        self.bos_index = self.add_token(bos_token)\n",
        "        self.eos_index = self.add_token(eos_token)\n",
        "\n",
        "        for token in tokens:\n",
        "            self.add_token(token)\n",
        "\n",
        "    def add_token(self, token):\n",
        "\n",
        "        self.word2idx[token] = new_index = len(self.word2idx)\n",
        "        self.idx2word[new_index] = token\n",
        "\n",
        "        return new_index\n",
        "\n",
        "    def encode(self, tokens):\n",
        "        # Input will look like:\n",
        "        # [<s>, w1, w2, ..., wn, </s>]\n",
        "\n",
        "        sequence = [self.bos_index]\n",
        "        for token in tokens:\n",
        "\n",
        "            index = self.word2idx.get(token, self.unk_index)\n",
        "            sequence.append(index)\n",
        "        sequence.append(self.eos_index)\n",
        "\n",
        "        return sequence\n",
        "\n",
        "    def create_padded_tensor_with_lengths(self, sequences):\n",
        "        # Input: [[4, 2, 3], [5, 4, 2, 4, 6]]\n",
        "        # Output:\n",
        "        # Tensor\n",
        "        # 4 2 3 0 0\n",
        "        # 5 4 2 4 6\n",
        "\n",
        "        # Given a list of sequences, pad all to the same length\n",
        "        lengths = [len(sequence) for sequence in sequences]\n",
        "        max_seq_len = max(lengths)\n",
        "        tensor = torch.full((len(sequences), max_seq_len), self.pad_index, dtype=torch.long)\n",
        "\n",
        "        for i, sequence in enumerate(sequences):\n",
        "            for j, token in enumerate(sequence):\n",
        "                tensor[i][j] = token\n",
        "        \n",
        "        return tensor, lengths"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ddaoNV3WK_lO"
      },
      "source": [
        "### Preparing the sequencers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpBnm8_dK6Xc"
      },
      "source": [
        "w2v_sequencer = W2VSequencer(word2vec_weights)\n",
        "train_tags = list(set(chain.from_iterable([e[1] for e in train_data])))\n",
        "label_sequencer = Sequencer(train_tags)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0ywHjCLREqUr"
      },
      "source": [
        "### Dataset Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fktMWrWUEgwe"
      },
      "source": [
        "class TaggerDataset(Dataset):\n",
        "    def __init__(self, data, input_sequencer, label_sequencer):\n",
        "        self.data = data\n",
        "        # [(tokens, tags)]\n",
        "\n",
        "        self.input_sequencer = input_sequencer # Convert word tokens to list of integers\n",
        "        self.label_sequencer = label_sequencer # Convert tag tokens to list of integers\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        tokens, tags = self.data[index]\n",
        "\n",
        "        x = self.input_sequencer.encode(tokens) # Input: [string], Output: [ints]\n",
        "        y = self.label_sequencer.encode(tags) # Input: [string], Output: [ints]\n",
        "\n",
        "\n",
        "        return x, y\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mEtdw1MjKlbT"
      },
      "source": [
        "### Make Dataset and DataLoader Objects"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N6rTuQNDKh1m"
      },
      "source": [
        "train_dataset = TaggerDataset(train_data, w2v_sequencer, label_sequencer)\n",
        "dev_dataset = TaggerDataset(dev_data, w2v_sequencer, label_sequencer)\n",
        "test_dataset = TaggerDataset(test_data, w2v_sequencer, label_sequencer)\n",
        "\n",
        "def prepare_batch(batch, in_sequencer, out_sequencer):\n",
        "    # batch: [batch_len, (text, label)]\n",
        "    texts, labels = zip(*batch)\n",
        "    text_tensor, lengths = in_sequencer.create_padded_tensor_with_lengths(texts)\n",
        "    label_tensor, _ = out_sequencer.create_padded_tensor_with_lengths(labels)\n",
        "    return (text_tensor, lengths, label_tensor)\n",
        "\n",
        "# Loader iterator: which produces a list of batches\n",
        "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=4, collate_fn=lambda batch: prepare_batch(batch, w2v_sequencer, label_sequencer))\n",
        "valid_loader = torch.utils.data.DataLoader(dev_dataset, batch_size=4, collate_fn=lambda batch: prepare_batch(batch, w2v_sequencer, label_sequencer), shuffle=False)\n",
        "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=4, collate_fn=lambda batch: prepare_batch(batch, w2v_sequencer, label_sequencer), shuffle=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GZ0dINH7WwVV"
      },
      "source": [
        "## Model Classes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S59PCuH2WvzY"
      },
      "source": [
        "class SimpleLSTMTagger(nn.Module):\n",
        "    def __init__(self, output_size, pad_index, rnn_hidden_size=100, dropout_p=0.5, w2v_weights=None):\n",
        "        super(SimpleLSTMTagger, self).__init__()\n",
        "\n",
        "        self.pad_index = pad_index\n",
        "\n",
        "        # Twitter embeddings of size 400\n",
        "        self.embedding = nn.Embedding.from_pretrained(w2v_weights, freeze=False)\n",
        "        embed_dim = 400\n",
        "\n",
        "        # Single Layer LSTM, unidirectional\n",
        "        self.rnn = nn.LSTM(\n",
        "            input_size=embed_dim,\n",
        "            hidden_size=rnn_hidden_size,\n",
        "            bias=True\n",
        "        )\n",
        "\n",
        "        # Dropout layer applied after the embedding layers\n",
        "        self.dropout = nn.Dropout(dropout_p)\n",
        "\n",
        "        # Maps RNN hidden state to a vector of size num_labels\n",
        "        self.fc = nn.Linear(rnn_hidden_size, output_size)\n",
        "        \n",
        "\n",
        "    def forward(self, x, lengths):\n",
        "        # x size: [batch_size, seq_len]\n",
        "        \n",
        "        batch_size = x.size(0)\n",
        "        embed = self.dropout(self.embedding(x)) \n",
        "        #embed: [batch_size = 4, seq_len, embed_dim = 400]\n",
        "\n",
        "\n",
        "        packed_input = nn.utils.rnn.pack_padded_sequence(embed, lengths, batch_first=True, enforce_sorted=False)\n",
        "        # packed_input (PackedTensor)\n",
        "\n",
        "        output, (h_n, c_n) = self.rnn(packed_input)\n",
        "\n",
        "        seq_unpacked, _ = nn.utils.rnn.pad_packed_sequence(output, batch_first=True, padding_value=self.pad_index)\n",
        "        # output: [batch_size, seq_len, hidden_size]\n",
        "        \n",
        "\n",
        "        logits = self.fc(seq_unpacked)\n",
        "        # logits: [batch_size, seq_len, output_size]\n",
        "\n",
        "        return logits"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IbtP09soYWMq"
      },
      "source": [
        "## Model Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ue6cv2pxKweS"
      },
      "source": [
        "hidden_size=50\n",
        "output_size = len(label_sequencer.idx2word)\n",
        "\n",
        "lstm_clf = SimpleLSTMTagger(output_size, label_sequencer.pad_index, rnn_hidden_size=hidden_size, w2v_weights=torch.FloatTensor(word2vec_weights.vectors))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4BHC4bn8lI1E",
        "outputId": "8b32d66a-adf5-4e73-8d94-8f6abd39027d"
      },
      "source": [
        "lstm_clf.to(device)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SimpleLSTMTagger(\n",
              "  (embedding): Embedding(1000003, 400)\n",
              "  (rnn): LSTM(400, 50)\n",
              "  (dropout): Dropout(p=0.5, inplace=False)\n",
              "  (fc): Linear(in_features=50, out_features=29, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D4SRSaPXY_Hu"
      },
      "source": [
        "## Experiment Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WeLdqZAvY0Eb"
      },
      "source": [
        "learning_rate = 1e-2\n",
        "loss_function = nn.CrossEntropyLoss(ignore_index=label_sequencer.pad_index) # Avoid computing loss on padding tokens\n",
        "optimizer = optim.Adam(lstm_clf.parameters(), lr=learning_rate)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OZTjmxcWZZyc"
      },
      "source": [
        "## Train and Evaluation classes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hElaWIVOZW0_"
      },
      "source": [
        " def train(model, optimizer, loss_function, loader, device, log_every_n=10):\n",
        "        \"\"\"\n",
        "        Run a single epoch of training\n",
        "        \"\"\"\n",
        "\n",
        "        model.train() # Run model in training mode\n",
        "\n",
        "        loss_history = []\n",
        "        running_loss = 0.\n",
        "        running_loss_history = []\n",
        "\n",
        "        for i, batch in tqdm(enumerate(loader)):\n",
        "            batch_size = batch[0].shape[0]\n",
        "            optimizer.zero_grad() # Always set gradient to 0 before computing it\n",
        "\n",
        "            logits = model(batch[0].to(device), batch[1]) # __call__ model() in this case: __call__ internally calls forward()\n",
        "            # [batch_size, seq_len, num_classes]\n",
        "            loss = loss_function(logits.view(-1, logits.size(2)), batch[2].view(-1).to(device)) # Compute loss: Cross entropy loss\n",
        "\n",
        "            loss_history.append(loss.item())\n",
        "\n",
        "            \n",
        "\n",
        "            running_loss += (loss_history[-1] - running_loss) / (i + 1) # Compute rolling average\n",
        "\n",
        "            loss.backward() # Perform backprop, which will compute dL/dw\n",
        "\n",
        "            if log_every_n and i % log_every_n == 0:\n",
        "                print(\"Running loss: \", running_loss)\n",
        "\n",
        "            running_loss_history.append(running_loss)\n",
        "            \n",
        "            nn.utils.clip_grad_norm_(model.parameters(), 3.0) # We clip gradient's norm to 3\n",
        "\n",
        "            optimizer.step() # Update step: w = w - eta * dL / dW : eta = 1e-2 (0.01), gradient = 5e30; update value of 5e28\n",
        "\n",
        "        print(\"Epoch completed!\")\n",
        "        print(\"Epoch Loss: \", running_loss)\n",
        "        print(\"Epoch Perplexity: \", math.exp(running_loss))\n",
        "\n",
        "        # The history information can allow us to draw a loss plot\n",
        "        return loss_history, running_loss_history\n",
        "\n",
        "    \n",
        "def evaluate(model, optimizer, loss_function, loader, device, labels, log_every_n=10):\n",
        "    \"\"\"\n",
        "    Evaluate the model on a validation set\n",
        "    \"\"\"\n",
        "\n",
        "    model.eval() # Run model in eval mode (disables dropout layer)\n",
        "\n",
        "    batch_wise_true_labels = []\n",
        "    batch_wise_predictions = []\n",
        "\n",
        "    loss_history = []\n",
        "    running_loss = 0.\n",
        "    running_loss_history = []\n",
        "\n",
        "    with torch.no_grad(): # Disable gradient computation - required only during training\n",
        "        for i, batch in tqdm(enumerate(loader)):\n",
        "            # batch[0] shape: (batch_size, input_size)\n",
        "\n",
        "            logits = model(batch[0].to(device), batch[1]) # Run forward pass (except we don't store gradients)\n",
        "            # logits shape: (batch_size, num_classes)\n",
        "            \n",
        "            loss = loss_function(logits.view(-1, logits.size(2)), batch[2].view(-1).to(device)) # Compute loss\n",
        "            # No backprop is done during validation\n",
        "            \n",
        "            # Instead of using CrossEntropyLoss, you use BCEWithLogitsLoss\n",
        "            # BCEWithLogitsLoss - independently calculates loss for each class\n",
        "            \n",
        "\n",
        "            loss_history.append(loss.item())\n",
        "\n",
        "            running_loss += (loss_history[-1] - running_loss) / (i + 1) # Compute rolling average\n",
        "            \n",
        "            running_loss_history.append(running_loss)\n",
        "\n",
        "            # logits : [batch_size, seq_len, num_classes] and each of the values in logits can be anything (-infinity, +infity)\n",
        "            # Converts the raw outputs into probabilities for each class using softmax\n",
        "            probs = F.softmax(logits, dim=-1) \n",
        "            # probs shape: (batch_size, seq_len, num_classes)\n",
        "            # -1 dimension picks the last dimension in the shape of the tensor, in this case 'num_classes'\n",
        "            \n",
        "\n",
        "            # softmax vector: [[0.1, 0.2, 0.6, 0.1, 0.0], [0.9, 0.01, 0.01, 0.01, 0.07]]\n",
        "            # output tensor: [2, 0]\n",
        "            predictions = torch.argmax(probs, dim=-1) # Output predictions; Argmax picks the index with the highest probability among all the classes (choosing our most probable class)\n",
        "            # predictions shape: (batch_size, seq_len)\n",
        "\n",
        "\n",
        "            batch_wise_true_labels.append(batch[2].view(-1).tolist())\n",
        "            batch_wise_predictions.append(predictions.view(-1).tolist())\n",
        "    \n",
        "    # flatten the list of predictions using itertools\n",
        "    all_true_labels = list(chain.from_iterable(batch_wise_true_labels))\n",
        "    all_predictions = list(chain.from_iterable(batch_wise_predictions))\n",
        "\n",
        "    # Non padding indices\n",
        "    pad_index = labels.index('<pad>')\n",
        "\n",
        "    \n",
        "    non_padding_labels = [all_true_labels[i] for i in range(len(all_true_labels)) if all_true_labels[i] != pad_index]\n",
        "    non_padding_predictions = [all_predictions[i] for i in range(len(all_true_labels)) if all_true_labels[i] != pad_index]\n",
        "\n",
        "    print(\"Evaluation Loss: \", running_loss)\n",
        "    # Now we can generate a classification report\n",
        "    print(\"Classification report after epoch:\")\n",
        "    print(classification_report(non_padding_labels, non_padding_predictions, target_names=labels, labels=range(len(labels))))\n",
        "\n",
        "    return loss_history, running_loss_history"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uhc5pAzRaoUd"
      },
      "source": [
        "def run_training(model, optimizer, loss_function, train_loader, valid_loader, device, labels, n_epochs=10):\n",
        "        # Useful for us to review what experiment we're running\n",
        "        # Normally, you'd want to save this to a file\n",
        "        print(model)\n",
        "        print(optimizer)\n",
        "        print(loss_function)\n",
        "\n",
        "        train_losses = []\n",
        "        train_running_losses = []\n",
        "\n",
        "        valid_losses = []\n",
        "        valid_running_losses = []\n",
        "\n",
        "        for i in range(n_epochs):\n",
        "            loss_history, running_loss_history = train(model, optimizer, loss_function, train_loader, device, log_every_n=10)\n",
        "            valid_loss_history, valid_running_loss_history = evaluate(model, optimizer, loss_function, valid_loader, device, labels, log_every_n=10)\n",
        "\n",
        "            train_losses.append(loss_history)\n",
        "            train_running_losses.append(running_loss_history)\n",
        "\n",
        "            valid_losses.append(valid_loss_history)\n",
        "            valid_running_losses.append(valid_running_loss_history)\n",
        "\n",
        "        # Training done, let's look at the loss curves\n",
        "        all_train_losses = list(chain.from_iterable(train_losses))\n",
        "        all_train_running_losses = list(chain.from_iterable(train_running_losses))\n",
        "\n",
        "        all_valid_losses = list(chain.from_iterable(valid_losses))\n",
        "        all_valid_running_losses = list(chain.from_iterable(valid_running_losses))\n",
        "\n",
        "        train_epoch_idx = range(len(all_train_losses))\n",
        "        valid_epoch_idx = range(len(all_valid_losses))\n",
        "        # sns.lineplot(epoch_idx, all_losses)\n",
        "        sns.lineplot(train_epoch_idx, all_train_running_losses)\n",
        "        sns.lineplot(valid_epoch_idx, all_valid_running_losses)\n",
        "        plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "926c6c75c58b436693bd94d1938706fa",
            "3f6f17e1767b433ca7b3112730b43203",
            "145e8369af46408db4c15f8ec896ae3d",
            "14391856f15540ee8e72d99f1eead095",
            "8c08cb96ce2f43209c82e2935be373a7",
            "17cecdc3fb7c4ec5acbc1dbf489f270f",
            "9afe8f957b704af7b692f2e10e264a4a",
            "59725d85f2bd4cbc83cf7b3a7f4b3c29",
            "91109a153a9447b2910ab1e6b4e546d2",
            "3cd33e45a6d9451c92e18242266214e8",
            "763d1ce8825f4429bb0f7fc258eba8dc",
            "e664845f97db4e2e9e4dae87e846246b",
            "174ccc3c013844058cde1c0aa3f0dace",
            "002eb334b02b4b80a7c648fde57f4835",
            "bcf76552de574086ab162ef8f94b2552",
            "962812fa4cc942bb85ffac0ad8cabdd6",
            "53c1829b4fc84b2c9368d22c4c16f53d",
            "1cde1b7b4f894ef9a75e0603c82778ce",
            "c4b8dd176a2b490ca3ea95d22808b1b1",
            "cd21705f1f4c44b0b20e979a3f7e08aa",
            "2adcca2bc07f421d936a6fbee862a26b",
            "db99b541ce9340b9907f0b800ff66511",
            "9a9621caaa5942f7925ea758a0285799",
            "35e2c1ab8ac74f5492f31c77ce2dfe74",
            "a6278e2d14e1446494ea283fb8511065",
            "08c98dc05a3b44db9d00e4ebafee82a0",
            "0805935de46b4ba6b62f95853e181c14",
            "834c90ad3130486bba9253b501ad0597",
            "a601a14da8ea4de09e50874bd34fd6b2",
            "2696177531db45e2a96761477d214c56",
            "7b153df9f5b146c483d905209b6a83e4",
            "b44362021c024b57bc40929e86026643",
            "3e149ccfeb914251839a4f582d4294fa",
            "eeb2b123540948ada2dbec590dbe3ce2",
            "50a38bfdffe843c785209e74c7e0e27c",
            "1f7f967da2ab439790368cb495de86cf",
            "6e958fd6e3d3469ab3e8c02f2e390d50",
            "b610f15d937143c18a742fdefe7ee840",
            "ce73330c28f04b32859b39cbbe0ddd01",
            "f0bf53e5e768466db05fd23323cd5dbd",
            "8fb34bb1dfd2439ab5a75405f2dc6583",
            "955a22169b594cb191603957f097c935",
            "7b7903cea18d45e09091d2d905686967",
            "452265a65d0f436e97b155cb70faa597",
            "3d43ac2e21ac46cea2dcfe7ac65c4242",
            "3bc83167e7054a6d93f2018d6191e92d",
            "9b9c6ed8e82a4c9d8435ccd1abc967d3",
            "63e42c663ccc4de881f9e6bf39987ffe",
            "be49cf3c4ba749bba46c5a325eef1bed",
            "66db0538de204d07a054b795beed3aa0",
            "8e6f8cc90f03496a8c8e6c386234d838",
            "0317315bacdb4d349133743e17fc45c0",
            "b9aa50bac4b3414f936f381bf3c5383b",
            "9e75d90f73d14de6b4864b83e6d29da4",
            "f4f33e870d624c02b94a2645da6e34d0",
            "1194f583e2a940babec8c82bc3de224e",
            "95794444b54d4dba901db75a5049f746",
            "749f695dc539449c80204633b74da0b4",
            "21ffe1a7b70745f6a71f9319ee51dda2",
            "8f7b5790307248b799127e25d0513c4d",
            "3d23cb3673a942ca9c1efb25795e2e6a",
            "7199119089694dbaa0b81dd34dd296fb",
            "6c423c9497924a6da9c5baa59c0252fb",
            "6e14f3c2cd69466b80af70639c0450f1"
          ]
        },
        "id": "ywO6T2_cbTbX",
        "outputId": "79d2f67b-40cd-4bb3-949a-bba0bda9ba7d"
      },
      "source": [
        "label_list = [label_sequencer.idx2word[i] for i in range(len(label_sequencer.idx2word))]\n",
        "\n",
        "print(label_list)\n",
        "run_training(lstm_clf, optimizer, loss_function, train_loader, valid_loader, device, label_list, n_epochs=4)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['<pad>', '<unk>', '<s>', '</s>', '#', 'G', '&', 'E', 'O', 'N', 'S', '$', '@', ',', 'V', 'R', 'X', 'U', '~', 'A', 'Y', 'M', '!', 'Z', 'D', 'T', 'L', 'P', '^']\n",
            "SimpleLSTMTagger(\n",
            "  (embedding): Embedding(1000003, 400)\n",
            "  (rnn): LSTM(400, 50)\n",
            "  (dropout): Dropout(p=0.5, inplace=False)\n",
            "  (fc): Linear(in_features=50, out_features=29, bias=True)\n",
            ")\n",
            "Adam (\n",
            "Parameter Group 0\n",
            "    amsgrad: False\n",
            "    betas: (0.9, 0.999)\n",
            "    eps: 1e-08\n",
            "    lr: 0.01\n",
            "    weight_decay: 0\n",
            ")\n",
            "CrossEntropyLoss()\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "926c6c75c58b436693bd94d1938706fa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Running loss:  3.3523263931274414\n",
            "Running loss:  2.8384680314497515\n",
            "Running loss:  2.425273276510693\n",
            "Running loss:  2.0748917518123506\n",
            "Running loss:  1.8088690173335196\n",
            "Running loss:  1.6372912117079192\n",
            "Running loss:  1.4798964955767648\n",
            "Running loss:  1.374176279759743\n",
            "Running loss:  1.2961795878263167\n",
            "Running loss:  1.2193009172167095\n",
            "Running loss:  1.1559076120357703\n",
            "Running loss:  1.1156865326133936\n",
            "Running loss:  1.0788387828621984\n",
            "Running loss:  1.0455135616637368\n",
            "Running loss:  1.0171708232122112\n",
            "Running loss:  0.9845963160920619\n",
            "Running loss:  0.9565656587759163\n",
            "Running loss:  0.9425100030606256\n",
            "Running loss:  0.9168541657002595\n",
            "Running loss:  0.8941028684533705\n",
            "Running loss:  0.8725031351123881\n",
            "Running loss:  0.8579466775836542\n",
            "Running loss:  0.8452396090079219\n",
            "Running loss:  0.8286605993390603\n",
            "Running loss:  0.816165684293415\n",
            "\n",
            "Epoch completed!\n",
            "Epoch Loss:  0.8078149027824405\n",
            "Epoch Perplexity:  2.24300145176959\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "91109a153a9447b2910ab1e6b4e546d2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluation Loss:  0.4718734311257921\n",
            "Classification report after epoch:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       <pad>       0.00      0.00      0.00         0\n",
            "       <unk>       0.00      0.00      0.00         0\n",
            "         <s>       1.00      1.00      1.00       327\n",
            "        </s>       1.00      1.00      1.00       327\n",
            "           #       0.85      0.21      0.34        52\n",
            "           G       0.60      0.28      0.38        65\n",
            "           &       1.00      0.95      0.97        91\n",
            "           E       0.81      0.67      0.74        52\n",
            "           O       0.92      0.98      0.95       333\n",
            "           N       0.81      0.87      0.84       660\n",
            "           S       0.00      0.00      0.00         5\n",
            "           $       0.44      0.50      0.47        86\n",
            "           @       0.67      0.92      0.78       243\n",
            "           ,       0.93      0.95      0.94       500\n",
            "           V       0.93      0.91      0.92       751\n",
            "           R       0.78      0.78      0.78       209\n",
            "           X       0.00      0.00      0.00         4\n",
            "           U       0.49      0.59      0.53        91\n",
            "           ~       0.91      0.88      0.89       170\n",
            "           A       0.78      0.76      0.77       239\n",
            "           Y       0.00      0.00      0.00         0\n",
            "           M       0.00      0.00      0.00         0\n",
            "           !       0.73      0.84      0.78        99\n",
            "           Z       0.00      0.00      0.00         9\n",
            "           D       0.97      0.90      0.93       312\n",
            "           T       0.79      0.75      0.77        36\n",
            "           L       0.90      0.92      0.91        65\n",
            "           P       0.93      0.92      0.93       440\n",
            "           ^       0.79      0.65      0.71       311\n",
            "\n",
            "   micro avg       0.86      0.86      0.86      5477\n",
            "   macro avg       0.62      0.59      0.60      5477\n",
            "weighted avg       0.87      0.86      0.86      5477\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "53c1829b4fc84b2c9368d22c4c16f53d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Running loss:  0.3743709921836853\n",
            "Running loss:  0.45869607952508057\n",
            "Running loss:  0.4469598084688186\n",
            "Running loss:  0.44639371143233386\n",
            "Running loss:  0.43188528280432625\n",
            "Running loss:  0.42630396432736345\n",
            "Running loss:  0.4055662858681601\n",
            "Running loss:  0.397243791482818\n",
            "Running loss:  0.39749386207557025\n",
            "Running loss:  0.3933668258589703\n",
            "Running loss:  0.38865443923980897\n",
            "Running loss:  0.39212855757088283\n",
            "Running loss:  0.38800011790734695\n",
            "Running loss:  0.38713817508848586\n",
            "Running loss:  0.3870998827595239\n",
            "Running loss:  0.3805300059006705\n",
            "Running loss:  0.37729790693391946\n",
            "Running loss:  0.3796586733507485\n",
            "Running loss:  0.37305625291407457\n",
            "Running loss:  0.3688454157355882\n",
            "Running loss:  0.36589752573204887\n",
            "Running loss:  0.36533417992329165\n",
            "Running loss:  0.36501033164189967\n",
            "Running loss:  0.3630890017173766\n",
            "Running loss:  0.36014776384743924\n",
            "\n",
            "Epoch completed!\n",
            "Epoch Loss:  0.359564833238721\n",
            "Epoch Perplexity:  1.4327058129363506\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a6278e2d14e1446494ea283fb8511065",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluation Loss:  0.5010519082226402\n",
            "Classification report after epoch:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       <pad>       0.00      0.00      0.00         0\n",
            "       <unk>       0.00      0.00      0.00         0\n",
            "         <s>       1.00      1.00      1.00       327\n",
            "        </s>       1.00      1.00      1.00       327\n",
            "           #       0.81      0.40      0.54        52\n",
            "           G       0.62      0.38      0.48        65\n",
            "           &       1.00      0.96      0.98        91\n",
            "           E       0.85      0.67      0.75        52\n",
            "           O       0.94      0.96      0.95       333\n",
            "           N       0.78      0.89      0.83       660\n",
            "           S       0.00      0.00      0.00         5\n",
            "           $       0.51      0.42      0.46        86\n",
            "           @       0.61      0.97      0.75       243\n",
            "           ,       0.93      0.95      0.94       500\n",
            "           V       0.94      0.88      0.91       751\n",
            "           R       0.77      0.79      0.78       209\n",
            "           X       0.00      0.00      0.00         4\n",
            "           U       0.59      0.53      0.55        91\n",
            "           ~       0.92      0.89      0.91       170\n",
            "           A       0.75      0.80      0.77       239\n",
            "           Y       0.00      0.00      0.00         0\n",
            "           M       0.00      0.00      0.00         0\n",
            "           !       0.76      0.84      0.80        99\n",
            "           Z       0.00      0.00      0.00         9\n",
            "           D       0.96      0.90      0.93       312\n",
            "           T       0.79      0.75      0.77        36\n",
            "           L       0.92      0.92      0.92        65\n",
            "           P       0.94      0.92      0.93       440\n",
            "           ^       0.86      0.59      0.70       311\n",
            "\n",
            "   micro avg       0.86      0.86      0.86      5477\n",
            "   macro avg       0.63      0.60      0.61      5477\n",
            "weighted avg       0.87      0.86      0.86      5477\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3e149ccfeb914251839a4f582d4294fa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Running loss:  0.27783480286598206\n",
            "Running loss:  0.3138223060152747\n",
            "Running loss:  0.3019478761014484\n",
            "Running loss:  0.3003305336160045\n",
            "Running loss:  0.2938266847191788\n",
            "Running loss:  0.28975162845031893\n",
            "Running loss:  0.2764220354986973\n",
            "Running loss:  0.2737153069653982\n",
            "Running loss:  0.2729126779753486\n",
            "Running loss:  0.2754240425733421\n",
            "Running loss:  0.27700231261182545\n",
            "Running loss:  0.2814253998232318\n",
            "Running loss:  0.27994698593931755\n",
            "Running loss:  0.28218305770677465\n",
            "Running loss:  0.2847294275853651\n",
            "Running loss:  0.2816517593844837\n",
            "Running loss:  0.28049394044076437\n",
            "Running loss:  0.28310234356693365\n",
            "Running loss:  0.27927315961672455\n",
            "Running loss:  0.27659181850664905\n",
            "Running loss:  0.2753372306365576\n",
            "Running loss:  0.2767499935464554\n",
            "Running loss:  0.27787796746754007\n",
            "Running loss:  0.27831224785932224\n",
            "Running loss:  0.2766639750095325\n",
            "\n",
            "Epoch completed!\n",
            "Epoch Loss:  0.276767204374075\n",
            "Epoch Perplexity:  1.3188593106164626\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8fb34bb1dfd2439ab5a75405f2dc6583",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluation Loss:  0.5334269836908433\n",
            "Classification report after epoch:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       <pad>       0.00      0.00      0.00         0\n",
            "       <unk>       0.00      0.00      0.00         0\n",
            "         <s>       1.00      1.00      1.00       327\n",
            "        </s>       1.00      1.00      1.00       327\n",
            "           #       0.74      0.48      0.58        52\n",
            "           G       0.60      0.40      0.48        65\n",
            "           &       1.00      0.93      0.97        91\n",
            "           E       0.85      0.65      0.74        52\n",
            "           O       0.91      0.95      0.93       333\n",
            "           N       0.73      0.90      0.81       660\n",
            "           S       1.00      0.20      0.33         5\n",
            "           $       0.42      0.41      0.41        86\n",
            "           @       0.76      0.90      0.82       243\n",
            "           ,       0.90      0.95      0.93       500\n",
            "           V       0.92      0.90      0.91       751\n",
            "           R       0.81      0.80      0.80       209\n",
            "           X       0.00      0.00      0.00         4\n",
            "           U       0.46      0.77      0.57        91\n",
            "           ~       0.94      0.81      0.87       170\n",
            "           A       0.82      0.72      0.76       239\n",
            "           Y       0.00      0.00      0.00         0\n",
            "           M       0.00      0.00      0.00         0\n",
            "           !       0.82      0.85      0.84        99\n",
            "           Z       0.00      0.00      0.00         9\n",
            "           D       0.94      0.89      0.92       312\n",
            "           T       0.77      0.67      0.72        36\n",
            "           L       0.92      0.92      0.92        65\n",
            "           P       0.92      0.92      0.92       440\n",
            "           ^       0.89      0.50      0.64       311\n",
            "\n",
            "   micro avg       0.86      0.86      0.86      5477\n",
            "   macro avg       0.66      0.60      0.62      5477\n",
            "weighted avg       0.86      0.86      0.85      5477\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "be49cf3c4ba749bba46c5a325eef1bed",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Running loss:  0.23919524252414703\n",
            "Running loss:  0.2655699937181039\n",
            "Running loss:  0.2595832922628948\n",
            "Running loss:  0.2580618307955804\n",
            "Running loss:  0.24764329576637692\n",
            "Running loss:  0.24132918055151026\n",
            "Running loss:  0.23476829428653248\n",
            "Running loss:  0.23165748646141776\n",
            "Running loss:  0.2295034720022001\n",
            "Running loss:  0.2327528579140102\n",
            "Running loss:  0.2331954368786646\n",
            "Running loss:  0.23761534392162478\n",
            "Running loss:  0.23476722858045707\n",
            "Running loss:  0.2348122974104098\n",
            "Running loss:  0.23757014072216145\n",
            "Running loss:  0.23460392355425466\n",
            "Running loss:  0.2346173479042438\n",
            "Running loss:  0.23631121786801434\n",
            "Running loss:  0.23328979173417908\n",
            "Running loss:  0.23086728202422874\n",
            "Running loss:  0.23061060075143078\n",
            "Running loss:  0.23092405061939306\n",
            "Running loss:  0.23273386197729343\n",
            "Running loss:  0.23355511180830718\n",
            "Running loss:  0.23379471646119443\n",
            "\n",
            "Epoch completed!\n",
            "Epoch Loss:  0.23328327524662013\n",
            "Epoch Perplexity:  1.2627391314067453\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "95794444b54d4dba901db75a5049f746",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluation Loss:  0.5464595596237884\n",
            "Classification report after epoch:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       <pad>       0.00      0.00      0.00         0\n",
            "       <unk>       0.00      0.00      0.00         0\n",
            "         <s>       1.00      1.00      1.00       327\n",
            "        </s>       1.00      1.00      1.00       327\n",
            "           #       0.64      0.48      0.55        52\n",
            "           G       0.52      0.35      0.42        65\n",
            "           &       1.00      0.92      0.96        91\n",
            "           E       0.77      0.71      0.74        52\n",
            "           O       0.92      0.95      0.94       333\n",
            "           N       0.78      0.87      0.83       660\n",
            "           S       1.00      0.60      0.75         5\n",
            "           $       0.45      0.51      0.48        86\n",
            "           @       0.80      0.88      0.84       243\n",
            "           ,       0.93      0.94      0.94       500\n",
            "           V       0.93      0.89      0.91       751\n",
            "           R       0.80      0.75      0.77       209\n",
            "           X       0.00      0.00      0.00         4\n",
            "           U       0.44      0.71      0.54        91\n",
            "           ~       0.91      0.88      0.90       170\n",
            "           A       0.75      0.78      0.77       239\n",
            "           Y       0.00      0.00      0.00         0\n",
            "           M       0.00      0.00      0.00         0\n",
            "           !       0.80      0.77      0.78        99\n",
            "           Z       0.00      0.00      0.00         9\n",
            "           D       0.94      0.90      0.92       312\n",
            "           T       0.83      0.67      0.74        36\n",
            "           L       0.92      0.91      0.91        65\n",
            "           P       0.88      0.94      0.91       440\n",
            "           ^       0.86      0.60      0.71       311\n",
            "\n",
            "   micro avg       0.86      0.86      0.86      5477\n",
            "   macro avg       0.65      0.62      0.63      5477\n",
            "weighted avg       0.87      0.86      0.86      5477\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
            "  FutureWarning\n",
            "/usr/local/lib/python3.6/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
            "  FutureWarning\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xcdZ3/8ddnLslM7kmTNkmb0iv0BhSI5SoipVDQBVHWFXyoeNmqq+KF/e2K/h74kN/q6urCyuKiCHhf8QIqIoIIyJ1CWwu9Qm/0mrZp2iTNPTPz/f1xTto0pE3aZjKdM+/n4zGPzJz5Zs7n5LTv+c73fM8Zc84hIiLZL5TpAkREZGQo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCCGDHQzi5nZS2b2ipmtMrOvDtLmejNrNLPl/u1j6SlXREQOJzKMNt3Axc65NjOLAs+a2Z+ccy8OaPdL59ynR75EEREZjiED3XlnHrX5D6P+TWcjiYicYIbTQ8fMwsBSYBrwXefc4kGavcfMLgReBz7vnNt6pNesrKx0kyZNOspyRURy29KlS/c456oGe86O5tR/MysDfgt8xjm3st/yMUCbc67bzD4O/INz7uJBfn8RsAhg4sSJZ23evPnotkREJMeZ2VLnXP1gzx3VLBfnXDPwJLBwwPIm51y3//Bu4KzD/P5dzrl651x9VdWgbzAiInKMhjPLpcrvmWNmcWABsHZAm5p+D68E1oxkkSIiMrThjKHXAD/2x9FDwK+ccw+Z2S3AEufcg8ANZnYlkAD2Atenq2ARERncUY2hj6T6+nq3ZMmSjKxbRCRbjdgYuoiInLgU6CIiAaFAFxEJiKwL9Nd27ufbj77G3vaeTJciInJCybpA39jYxh1PrqehpTPTpYiInFCyLtCLY1EA2roSGa5EROTEkoWB7k2d369AFxE5RPYGendvhisRETmxZGGge0Mu6qGLiBwqCwNdQy4iIoPJukCPRcPkhUO0dmnIRUSkv6wLdPB66eqhi4gcSoEuIhIQWRroUfZryEVE5BBZGugRWjsV6CIi/WVloI8pyte1XEREBsjKQK8qyqdxf/fQDUVEckh2BnpxPu09Sdq7dWBURKRPVgb62OJ8APa0qZcuItInKwO9yg90DbuIiBykQBcRCYisDPTKIj/QNeQiInLAkIFuZjEze8nMXjGzVWb21UHa5JvZL81svZktNrNJ6Si2T3mBd8XFpjZNXRQR6TOcHno3cLFz7nRgLrDQzM4Z0OajwD7n3DTgNuCbI1vmoSLhEOUFUc1FFxHpZ8hAd542/2HUv7kBza4Cfuzf/w0w38xsxKocREVhngJdRKSfYY2hm1nYzJYDu4HHnHOLBzQZD2wFcM4lgBZgzEgWOtCYwnxNWxQR6WdYge6cSzrn5gITgHlmNudYVmZmi8xsiZktaWxsPJaXOGBMkXroIiL9HdUsF+dcM/AksHDAU9uBOgAziwClQNMgv3+Xc67eOVdfVVV1bBX7NOQiInKo4cxyqTKzMv9+HFgArB3Q7EHgQ/79a4AnnHMDx9lHVFVxPns7eujqTaZzNSIiWWM4PfQa4EkzexV4GW8M/SEzu8XMrvTb3AOMMbP1wBeAL6an3INm1ZTgHKza0ZruVYmIZIXIUA2cc68CZwyy/OZ+97uAvx/Z0o5sRnUJABsa2zjrpPLRXLWIyAkpK88UBagpixEOGVv3dmS6FBGRE0LWBno0HKK2LMbmJgW6iAhkcaADTKwoYIt66CIiQNYHeiFvNLWT5gk1IiJZIasDfVZNMc0dvWxv7sx0KSIiGZfVgT5nfCkAK7dr6qKISFYH+syaEkIGqxsU6CIiWR3osWiYscUxdmjIRUQkuwMdoLo0xs6WrkyXISKScVkf6DWlMRpa1EMXEcn6QK8ujbGrVddFFxHJ/kAvidHWnWB/V2+mSxERyajsD/TSGIDG0UUk52V9oNeUxgFoUKCLSI4LQKCrhy4iAgEI9LEl+QDs0EwXEclxWR/o+ZEwVcX5NDSrhy4iuS3rAx2gtiyuHrqI5LxABPr4spiuuCgiOS8QgV5bGmdHc6euiy4iOS0YgV4Wp6s3xb4OnVwkIrkrMIEO6KqLIpLThgx0M6szsyfNbLWZrTKzzw7S5iIzazGz5f7t5vSUO7jaMm8uugJdRHJZZBhtEsCNzrllZlYMLDWzx5xzqwe0e8Y5986RL3Fo6qGLiAyjh+6ca3DOLfPv7wfWAOPTXdjRGFOYR14kxA6dLSoiOeyoxtDNbBJwBrB4kKfPNbNXzOxPZjZ7BGo7mroYXxbX1EURyWnDGXIBwMyKgPuBzznnBn6J5zLgJOdcm5ldAfwOmD7IaywCFgFMnDjxmIseTG1ZjAYFuojksGH10M0sihfmP3fOPTDweedcq3Ouzb//MBA1s8pB2t3lnKt3ztVXVVUdZ+mH8uaia8hFRHLXcGa5GHAPsMY5d+th2lT77TCzef7rNo1koUOpLYuza38XvcnUaK5WROSEMZwhl/OBDwArzGy5v+xLwEQA59z3gGuAT5pZAugE3udG+bTN2rIYznmX0a2rKBjNVYuInBCGDHTn3LOADdHmDuCOkSrqWPRNXdze3KlAF5GcFIgzRQEmVxYCsLGxPcOViIhkRmACvbY0TjwaZv3utkyXIiKSEYEJ9FDImFAeZ3tzR6ZLERHJiMAEOsD4cp1cJCK5K1CBXlumuegikrsCFejjy+Lsbe+hsyeZ6VJEREZd4AId0LCLiOSkQAW6LqMrIrksUIE+vlw9dBHJXYEK9HHF+URCxta9mrooIrknUIEeCYeYOKZAZ4uKSE4KVKADTKksYkOjzhYVkdwTuECfWlXI5qYOkqlRvdijiEjGBS7Qp1QV0pNMsW2fxtFFJLcEMNCLANi4R+PoIpJbghfouoyuiOSowAV6RWEeJbEIG3VgVERyTOAC3cyYUlXEJg25iEiOCVygg3dgVEMuIpJrghnolYXsbO2ivTuR6VJEREZNMAPdn+miYRcRySUBDXR/posCXURyyJCBbmZ1Zvakma02s1Vm9tlB2piZ3W5m683sVTM7Mz3lDs+kMYWYwSaNo4tIDokMo00CuNE5t8zMioGlZvaYc251vzaXA9P929nAnf7PjIhFw9SWxtm4R1MXRSR3DNlDd841OOeW+ff3A2uA8QOaXQX8xHleBMrMrGbEqz0KmukiIrnmqMbQzWwScAaweMBT44Gt/R5v482hP6qmVhWxsbEN53SRLhHJDcMOdDMrAu4HPuecaz2WlZnZIjNbYmZLGhsbj+Ulhu3kccW09yT17UUikjOGFehmFsUL85875x4YpMl2oK7f4wn+skM45+5yztU75+qrqqqOpd5hO3mcN3Xx9V3707oeEZETxXBmuRhwD7DGOXfrYZo9CHzQn+1yDtDinGsYwTqP2vRxxQCs3alAF5HcMJxZLucDHwBWmNlyf9mXgIkAzrnvAQ8DVwDrgQ7gwyNf6tEpjUepLY3xmgJdRHLEkIHunHsWsCHaOOBTI1XUSJlZU8KahmMa7hcRyTqBPFO0z4yaYjY0ttPVm8x0KSIiaRfoQJ9dW0oy5XRgVERyQqAD/dTxpQC8sq0lw5WIiKRfoAN9Qnmc8oIoK7Y1Z7oUEZG0C3Sgmxmn15Xxty0KdBEJvkAHOsBZE8tZt7uNlo7eTJciIpJWwQ/0SeUALNuyL8OViIikV+ADfW5dGeGQsWTz3kyXIiKSVoEP9IK8CLNrS1i6WT10EQm2wAc6wJkTy1m+tZneZCrTpYiIpE1OBHr9pHK6elOs3qHLAIhIcOVGoJ9UAcASDbuISIDlRKBXl8aYNKaAp15P75dqiIhkUk4EOsDCOTU8v34PzR09mS5FRCQtcibQ33FqDYmU48+rdmW6FBGRtMiZQJ8zvoS6ijh/XJHRL1ISEUmbnAl0M+OKU2t4TsMuIhJQORPoAFfM8YddVmvYRUSCJ6cC/bQJpUwoj/Poyp2ZLkVEZMTlVKCbGW+dXslLb+ylJ6GzRkUkWHIq0AEunV3N/q4Ef31td6ZLEREZUTkX6BdMq6QwL8wz6/ZkuhQRkRE1ZKCb2b1mttvMVh7m+YvMrMXMlvu3m0e+zJETDYc4e8oYnlnXiHMu0+WIiIyY4fTQfwQsHKLNM865uf7tluMvK73mzxzLG00drG7QxbpEJDiGDHTn3NNAoL4d4oo5NURCxoOv7Mh0KSIiI2akxtDPNbNXzOxPZjZ7hF4zbcoL83jr9Er+sHwHqZSGXUQkGEYi0JcBJznnTgf+G/jd4Rqa2SIzW2JmSxobM3vlwyvn1rKjpYul+q5REQmI4w5051yrc67Nv/8wEDWzysO0vcs5V++cq6+qqjreVR+XBbOqyY+EeHC5hl1EJBiOO9DNrNrMzL8/z3/NpuN93XQryo+wYNY4/vDqDroTyUyXIyJy3IYzbfEXwAvAKWa2zcw+amafMLNP+E2uAVaa2SvA7cD7XJbMB3xvfR3NHb26pK6IBEJkqAbOuWuHeP4O4I4Rq2gUXTCtkgnlcW597HUum11NXiTnzrMSkQDJ6QQLhYyvXjmbTXva+fnizZkuR0TkuOR0oANcPGMs50yp4FuPvsb+rt5MlyMicsxyPtDNjC9ePpOOniS/WrIt0+WIiByznA90gNMnlHLe1DHc9tjrrNjWkulyRESOiQIdr5f+jXefRnEswr/e/6ou2iUiWUmB7ps4poAbLz2F1Q2t/PLlrZkuR0TkqCnQ+3n3GeM5Z0oFX/vjGnbv78p0OSIiR0WB3k8oZHz96lPp6E1y11MbM12OiMhRUaAPMKWqiKtOr+Xni7ewvbkz0+WIiAybAn0Qn7vkZMIh459/9YoOkIpI1lCgD8I7QHoyL2xs4rn1J/x1xkREAAX6YV07byJ1FXH+7+9WsLe9J9PliIgMSYF+GLFomP931Ry27O3gg/cupqtXl9gVkRObAv0ILjplLP/z/rNYtaOVmx5YofF0ETmhKdCHsHBONV+45GR++7ft3PvcG5kuR0TksBTow/Cpt0/jstnj+PrDa3j5jb2ZLkdEZFAK9GEIhYz/fO9cakpjfPiHL/PIyoZMlyQi8iYK9GEqyo/wy4+fy9SqQj7xs2V87Y+rSaY0pi4iJw4F+lEYXxbn1584j+vOnsgPntnEp36+jI6eRKbLEhEBFOhHLS8S4mvvmsOXr5jJo6t38q7vPsfijTr5SEQyT4F+DMyMf7xwCj/+8DzauhJcd/di/nfxlkyXJSI5ToF+HC48uYo/f+FtXDCtki/9dgVfvP9VnYAkIhkzZKCb2b1mttvMVh7meTOz281svZm9amZnjnyZJ66i/Aj3Xv8W/umiqdz38lbm3vJnfvjcJlI6YCoio2w4PfQfAQuP8PzlwHT/tgi48/jLyi7hkPEvC2dw9wfrKY5F+eofVnPF7c/w6rbmTJcmIjlkyEB3zj0NHOlsmquAnzjPi0CZmdWMVIHZ5JJZ41h803z+45rTaGrv4arvPseXfruC7oSGYUQk/UZiDH080P9LOLf5y3JSKGS8t76Ox298Gx8+bzL/u3gLV3/3eZ7fsCfTpYlIwI3qQVEzW2RmS8xsSWNj42iuetSVxKLc/HezuO0fTqels5frfrCYj/90CVuaOjJdmogE1EgE+nagrt/jCf6yN3HO3eWcq3fO1VdVVY3Aqk98V58xgcdvfBv/57JTeGbdHi77r6e56YEVbNvXoas3isiIiozAazwIfNrM7gPOBlqcc7rYST+xaJhPvX0aV58xnv94ZC33L9vGL17aQl1FnAUzq/nAuScxubIw02WKSJazoXqJZvYL4CKgEtgFfAWIAjjnvmdmBtyBNxOmA/iwc27JUCuur693S5YM2SyQtjR18Jtl2/jbln08s24PsWiIz19yMh86bxKxaDjT5YnICczMljrn6gd9LlMf+3M50Pvb1drFl3+7kr+s2cWYwjzeN6+O684+ifFl8UyXJiInIAX6Cc45x4sb93LPsxt5Yu1uAC6ZOY4b5k9nzvjSDFcnIicSBXoW2bavg1+8tIWfvbiFls5eLp9TzfXnTWLe5Aq80S0RyWUK9CzU0tnL3c9s5EfPvcH+7gQTKwqYM76EK08fz/yZY4mGdRkekVykQM9inT1Jfr98O39auZNXtzWzr6OXyqI8LpxexVtPruT8aZWMLY5lukwRGSUK9IBIJFM89Xojv1u+g2fXNbKvoxeA2bUlXHPWBC6ZOY66ioIMVyki6aRAD6BUyrG6oZWn1zXy0CsNrG5oBeCUccXMGV/KuVPHUH9SORMrCgiFNPYuEhQK9BywaU87j6/ZxV9fa2Ttzlb2tPUAUF4Q5bypldxx3Rk6qCoSAAr0HJNKOdbsbGXFthZu/v0qepIp1tyykHieTloSyXZHCvSROPVfTjChkDG7tpTZtaW0dvXy9YfXktJ1Y0QCT3PfAi7kD7MkFegigadAD7i+QNdX4okEnwI94ML+DJekAl0k8BToAdc3ZVFDLiLBp0APuPCBIZcMFyIiaadAD7i+S76ohy4SfAr0gNNBUZHcoUDPpN1rYd1jkOhJ2ypiiRaMlA6KiuQABXqmdDbD/5wNP78GXvyf9KyjYy9/98j53Bj5tYZcRHKAAj1Tkv165V3N6VlH934A/iH8Vw25iOQABXqmpBIH7/d2pWcdIe/aLVXWQnzTY+lZh4icMBToR9LwKiQTQ7c7Fsneg/fbdqZnHf3eNAq2PNFv3QloeMWby5joBg3HiASCAv1wdq2C778Vnv6P9Lx+/x76qt9Cd5t3f+dKeOpbsOlpWPEbaGs89nX0ezOyhP8pwDl44hb4/oVwSzn821j47Se857a8CLfOhke/DF2tx75eEcmIYV1t0cwWAt8BwsDdzrlvDHj+euBbwHZ/0R3OubtHsM7R1dsFd57n3d++ND3r6N9DB7j9DHjvj+GHlx+6PL8UPrPEC9invgFz3w9TLoLhXNu835tG+eu/hj9XQ1cLLPvJoe1evQ82PAHtu73HL9wBr9wHNadBrAxKaqG0DqIxaNsNeUVw8mXQ0w5//QZMOh/O/iSE1D8QyaQhA93MwsB3gQXANuBlM3vQObd6QNNfOuc+nYYaR1+i8+D9SL/v60yloPkNKJ88vEA9ktSAQG/ffWiYj5sDu1ZCdwt8e/rB5St+DaUTYeI5UDzO63EXVkF7ox+2hVBzOuxvgKe+eeg6nv/vg/fffz9Mvdh7/XsuhT2vw/TL4JKvQOc+ePLfvZA/nEdvOng/2Q3n/NPR/w1EZEQNp4c+D1jvnNsIYGb3AVcBAwM9OFLJg/c3PuUF+fKfwYOfObTdtffBKZd749DLfuIF5Jipw1tHXw/93T+Awkr46dXe45lXwhXfguJq7/Hzd8BfvgJzr/Nev60RnvsvWPGrwV83rwiW/nDw5ybMg20v0fWZlcTG1HnL4uVw/cNeL/0t/+j1wgGufwiaN8PWl2DGO6FlG2xfAgVjoHK6N0zU2wnnfgrCecf/Bicix23Ibywys2uAhc65j/mPPwCc3b837g+5/DvQCLwOfN45t/VIr3tCf2NR6w64debw2kYLvPDduxEsBPM+Dm/5GOxaAe17IBSBfZu8IE50QVmdF4BPf8v7/fffD9Mv8XrF6/4Cp17z5nDs6YC8fl/+3NsFr/8J9qzz1uXcwamP5ZNg83Nem4f/GZo38/Gez1F+ygVc+/Z6PvuzF0iEYzz7rxcf959JREbfaHxj0R+AXzjnus3s48CPgTclhpktAhYBTJw48djXluiBP3wW3voFr7cIXm+xdQfsewM2/hUu+DwUVHi97RW/gZPOhTJ/nf3fxBJdXvD2dnphGMnzlg0UzvfGuKct8KYDbnsZ7lkAvR1eqF/6NWhaD4u/B4vvHLzu4lpY9cCA1/V3QbwcTvv7wX+vf5iD14ueffWhywrHHLw/+ULv5yPetMV24jy6pof71jzvN+jk3mc38ZELJg++PhHJSsMJ9O1AXb/HEzh48BMA51xTv4d3A4NODXHO3QXcBV4P/agq7ZNKwb9Vefdbt8OHHvSGA26bfWi752+HC74AWxd7PVbwhhzGzfaGJCIxiMa9nnGfUBQmvAW2+MF39ie8ceTL/t078BeNH2xbNw++vBPeeBamzj94QPAtH4O//cwb0563CFzSG+OOl3tDK+1N0LkXfvZuaN7irTNdzKspOchkpm88spaFc6qpLYu/6TkRyU7DCfSXgelmNhkvyN8HXNe/gZnVOOca/IdXAmtGtMr+ejsO3t/0lDfT4gfzvcflk+DMD8Ibz3kh/uyt3pDH9Mtg/Fmw9qGD48sVU6F2LkTyvR72uDleOO9aefD16+bB5QMOLPYXjcP0BYcuq54Dl39j8Pbg9aQLx3jDLgDh9Ad6wh365dA3zJ/O95/awKKfLuFLl8+kJB7lTysbKIvn8ff1EygryEtfTSKSNkMGunMuYWafBh7Fm7Z4r3NulZndAixxzj0I3GBmVwIJYC9wfdoqHjg75Ou13s/5N3s9cjN4643e9Lzl/+sdTIyVem0u+teDJ9WMP/PNY9Vzr/V+/uBib7piOnvPfYEeSuP3dJsX5Am8nx8+fxLdiRT/dNFUqorzufn3K7nu7sWH/MrXHl7DvMkVfPmKmWxv7uSFDU1cML2SS2eNw3TgU+SENqw0cc49DDw8YNnN/e7fBNw08PfSYuD8bYB33Ar1Hzk0oGOlcM4n39w2HIEJZx15HX7PNq295wOvncazNPt66H6gv+fMCcwZ7725feCck3jnqTXc9/JWUs5x8YyxvLChiW//+TVe2rSXq7773IGX+emLm3nr9Eq+fvWp1FUUvHk9InJCSGP3ME36Lmo1453eSS9nL/LmXY+kUQl0v4c+2BvUSPG3I4X3RheLHjqWXl6YxycvOjjNcmZNCR+5YDItnb38+Pk3qCjMY0Z1MY+s3MmPnn+D+f/5FJefWs30sUX0JLyvQNrQ2E5dRQFnT6lgYkUBUyoL6ehJEg4Z+RFvfe09SUIG0XCIaFgnH4mkSxYGuh+AM955cIhkpPlDFWkdcuk7wJpK07Vi4MCB2pD/KSAvHD5S6wNK41FumH/wZKb6SRV86LxJ3PnUBu5fuo3fJ978fXbfe2oDAAV5YTp6vHn8sWgI56Dbbx/xQ74kHmXa2CJqSmO0diZo7uxh9/5uimNRqkvymVFdwtiSfCZWFFBXXsC+jh5K4lHauxM0tHRRUZjH3LoyvTmIDJC9gZ7O3nNoeMF3XN51Jzx7G9Sdk751+G9MYbxAzY8eewDWVRTw9atP5f9cegrtPQmK86OknKO8MI/27gQvbGhixfYWVm5vobYsTkVhHvs6emhq62FWbQnJlKM7kaSzJ0VDSyc7WrpYu7ORlo5eJlcWMq2qiLbuBOt3t/Hn1buGvF5YOGQUxyIU5UeIhkNUFecztjifkniUsniUcSUx2roThENGUb7XLuUceZEQhXkRYtEwkbARj4YpiUUJhSASChHPC1OUH2FHcyfhkGkWkGSVLAx0f8hlFGaH4JJHbnc8SifAO/4zfa8PB7Yj5Ad63gj0aMsL8ygvPHQWTGF+hEtmjeOSWeOO+vVSKUcodOjB1o6eBC2dvWxu6mBzUzulce9NoyQeJRIyOnuTrNrRQltXgv3dCbp6k+xu7WbVjlb2d/XS3NFL4jiu/16UH6Gt2/vk9NjnL2T3/m7On1Z5zK8nMlqyONDTOLWur4eeevPQQlapngPbl9BCIXB8PfR0GRjmAAV5EQryItSUxjlnyphBfguuOLXmsK+ZTDn2tveQHw0RNqO9O0Fbd4KQGT3JFO3dCTp7kyRTzn8uSSrlSKQcj6/ZxeNrdx94rfffvZjd+7v5zMXT+MKCkzXTR05o2RfofWPO2d5DHw0Lv0ly9nvYcJf3zUUj0UPPBuGQUVWcf+BxYX6EscP83TnjSw4J9N37uwH47yfWs2DWOE6bUDaSpQZGKuVweH97yZzsC/TR6KH3XRir/5UWs1E0RnjKhcAfAYjkSKAfj6L8N/+XuGTmOF7YsId/+c2rnD+tkotnjOUva3axblcbU6oKefspY7nolKqc7r1/9pfL+cMrO1h+8wJWN7Ry3lQNUWVC9gZ6OmegLPymd2Zp3zVRJGcUxw7+u/rme04lFg1z0SljeXbdHm6472+s3bmfe57ddKDNs+v38JMXNjOhPE55gXcgeEZ1Me86YzxvO7nqkNcbinOOZMrR1N5DR0+S7kSSaDhEMuWoLo1REovS0tHLa7v207i/m55kkpAZ4ZBRUZBHaUGU0nj0wCyjprYeOnsTGMbJ1cVUFeXTm0wRj4YHHeo6Hn94ZQcAtzy0mgeWbeeG+dP53PzpI74eObIsDPS+WS5p7KHnF3knKgXEDz5YT3NHz9ANheLYwf8SVcX5XDzDO9D7jtNquOiUKv6yZhf5kRCnVJdQWZRHQ0sXj63excMrGsiPhJhRXcziTXv5y5rdxKIh3jKpgqlVRaxpaKUkHqWtK0EkbIwrieEcVBbnsWzzPt5o6mBvew8Ghz2gGw4ZyRH6su94NEx+1Jvxk+efL5Dy31Cc845DJJ2jpbOXaMgImfnHYIyQeefwGYYZB843AHhgmXeZp9sfX8ekMQW8+8wJ3mv3q1shnz5ZHOhp7KEHzIJjmH2Sq2LRg1NW49FD/3sU5ke4au74Q5YVx6KcPK6YT7192oFlyZTjqdd388Ta3SzeuJfnNzRRGo9SUZhHyCAcCrF8SzPdyRQ9iRQ1pTHOnlxBpd+DnlFdTGG+N7WyN5kiZMaO5k72dyVwOGbWlDBpTCGF+RGSqRQdPUnau5O0dPbS1N5NfiRMfiRERWGed0qZwaY97TS19ZAfCdHek6SjO0FXIklHT5LepBe2YYOQGaGQ+XUasWiYkBkp5+hOpPzppF7oO+e9+dy/bNshf5O3nVzFjuZOvvCrV7jpgRUsmDWOp19vpLUrwYzqYv7u9Fo+eO5JR/XpRYYnCwN9FMbQRYB43rGdjxAOGRfPGHegd++cG3R8vTeZoqmth3El+Wkff0/nmPYjKxto70lyw/zpvGtuLcWxKLtau/jKg6tYunnfgTCfNKaA5o5evvXoa3zn8XVUFOQRDhnjy+OcN3UMM6pLmFAeZ/H76bkAAAbPSURBVFZNCdv2dRKLhsiPhInlhYiGQgd69o37u9nV2kV3IkVrVy/b9nXS3p0g5RxVRfnE88KUxqMU+ucfbN3bQVN7D/u7EuSFjXhehIK88IH9292bIho2unpTdPUm6ehJMLmyiKJYhEjIG9KKR732sYj3yabvTa7vk0wq5Uj1fbLxlyWSKXbv7/bPkDZ6EinC/qedscX5jC0Z+WN02Rfo0y+Fzyzz5nGLpFHBMQb6QIcL62g4RHVplh94B+8s3p4kpfEoU6qKAG+46v5Pnjdo++Vbm/npC5tp7erFOdjc1M53Hl93xJPJYtEQhlFeEGVHyyDfV5BlPvG2qXzx8hkj/rrZF+j5Rd5NJM3i0VE4YzgAQv4bVv/jD0cyt66MuXWHTv/c3drFtuZONjW2s3JHC+NKYhTmR2jt7KUnkaKl0xtqXb+7jYtmjOXcKWMojnln/04fW0xRfoRI2Njb3kNnb5Lmjl46e5Lsau2iujRGXUUBFQV5JFKOzp4kHb0Jb7gpkcLhDbXlR0KknKM4P8r25k66epMkUl5Pu6MnSVciSVdviu5EEue84whh/6B0yA4OU4VDIcIh7418XEmMZCpFT8IdeP1EyjFpTOGI7oM+2RfoIqOk/1x2Oby+73YpHmTK53CNLYkxtiTGmRPLec9Zx/7pe6TG5SeOyc6rimpissgA7zythpPGFBxygFQOr7vXO6O6olDHtTJNPXSRAe647sxMl5BV+mYkzqguyWwhokAXkePz44+8hefW76G0QNMQM02BLiLHZXZtKbNrSzNdhqAxdBGRwFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQ5o50zcp0rtisEdh8jL9eCewZwXKygbY5N2ibc8PxbPNJzrmqwZ7IWKAfDzNb4pyrz3Qdo0nbnBu0zbkhXdusIRcRkYBQoIuIBES2BvpdmS4gA7TNuUHbnBvSss1ZOYYuIiJvlq09dBERGSDrAt3MFprZa2a23sy+mOl6RoqZ1ZnZk2a22sxWmdln/eUVZvaYma3zf5b7y83Mbvf/Dq+aWVZ+zY6Zhc3sb2b2kP94spkt9rfrl2aW5y/P9x+v95+flMm6j4eZlZnZb8xsrZmtMbNzg7yfzezz/r/plWb2CzOLBXE/m9m9ZrbbzFb2W3bU+9XMPuS3X2dmHzqaGrIq0M0sDHwXuByYBVxrZrMyW9WISQA3OudmAecAn/K37YvA48656cDj/mPw/gbT/dsi4M7RL3lEfBZY0+/xN4HbnHPTgH3AR/3lHwX2+ctv89tlq+8AjzjnZgCn421/IPezmY0HbgDqnXNzgDDwPoK5n38ELByw7Kj2q5lVAF8BzgbmAV/pexMYFudc1tyAc4FH+z2+Cbgp03WlaVt/DywAXgNq/GU1wGv+/e8D1/Zrf6BdttyACf4/8ouBhwDDO9kiMnB/A48C5/r3I347y/Q2HMM2lwKbBtYe1P0MjAe2AhX+fnsIuCyo+xmYBKw81v0KXAt8v9/yQ9oNdcuqHjoH/3H02eYvCxT/Y+YZwGJgnHOuwX9qJzDOvx+Ev8V/Af8CpPzHY4Bm51zCf9x/mw5sr/98i98+20wGGoEf+kNNd5tZIQHdz8657cC3gS1AA95+W0rw93Ofo92vx7W/sy3QA8/MioD7gc8551r7P+e8t+xATEsys3cCu51zSzNdyyiLAGcCdzrnzgDaOfgxHAjcfi4HrsJ7I6sFCnnzsEROGI39mm2Bvh2o6/d4gr8sEMwsihfmP3fOPeAv3mVmNf7zNcBuf3m2/y3OB640szeA+/CGXb4DlJlZ35eX99+mA9vrP18KNI1mwSNkG7DNObfYf/wbvIAP6n6+BNjknGt0zvUCD+Dt+6Dv5z5Hu1+Pa39nW6C/DEz3j5Dn4R1ceTDDNY0IMzPgHmCNc+7Wfk89CPQd6f4Q3th63/IP+kfLzwFa+n20O+E5525yzk1wzk3C249POOfeDzwJXOM3G7i9fX+Ha/z2WdeLdc7tBLaa2Sn+ovnAagK6n/GGWs4xswL/33jf9gZ6P/dztPv1UeBSMyv3P91c6i8bnkwfRDiGgw5XAK8DG4AvZ7qeEdyuC/A+jr0KLPdvV+CNHz4OrAP+AlT47Q1vxs8GYAXeLIKMb8cxbvtFwEP+/SnAS8B64NdAvr885j9e7z8/JdN1H8f2zgWW+Pv6d0B5kPcz8FVgLbAS+CmQH8T9DPwC7zhBL94nsY8ey34FPuJv/3rgw0dTg84UFREJiGwbchERkcNQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEP8fNUsu7DCTK/EAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "df178ac82b294bba8382a4954e13f064",
            "7ded50dfc1ba4a3190db316bce961bdf",
            "e8161483e9074fd18494c707cd3ce466",
            "b359439258574552b7034d3999c4f44e",
            "b55718ee707247b689a4806bb30440c7",
            "1799e75ffbd7429ca76b0ef1b22ce09e",
            "9c3728a64cbf41868f4c8e54b8c39994",
            "cadfda6c7314495d84b0e5a8366a8440"
          ]
        },
        "id": "Av6zRA3TbrQV",
        "outputId": "aaf6d7a6-a81d-4530-d924-98d3fee172b8"
      },
      "source": [
        "print(\"Evaluation on test set\")\n",
        "evaluate(lstm_clf, optimizer, loss_function, test_loader, device, label_list, log_every_n=10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Evaluation on test set\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "df178ac82b294bba8382a4954e13f064",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluation Loss:  0.5157939029932024\n",
            "Classification report after epoch:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       <pad>       0.00      0.00      0.00         0\n",
            "       <unk>       0.00      0.00      0.00         0\n",
            "         <s>       1.00      1.00      1.00       500\n",
            "        </s>       1.00      1.00      1.00       500\n",
            "           #       0.67      0.44      0.53        78\n",
            "           G       0.24      0.23      0.23        70\n",
            "           &       0.99      0.98      0.99       127\n",
            "           E       0.79      0.78      0.78        63\n",
            "           O       0.93      0.94      0.93       505\n",
            "           N       0.80      0.88      0.84       981\n",
            "           S       0.75      0.50      0.60         6\n",
            "           $       0.38      0.45      0.41        85\n",
            "           @       0.79      0.87      0.83       330\n",
            "           ,       0.95      0.95      0.95       880\n",
            "           V       0.94      0.89      0.91      1053\n",
            "           R       0.76      0.81      0.78       339\n",
            "           X       0.67      0.33      0.44         6\n",
            "           U       0.45      0.79      0.57       117\n",
            "           ~       0.94      0.86      0.90       212\n",
            "           A       0.78      0.77      0.77       367\n",
            "           Y       0.00      0.00      0.00         0\n",
            "           M       0.00      0.00      0.00         0\n",
            "           !       0.82      0.82      0.82       186\n",
            "           Z       0.00      0.00      0.00        22\n",
            "           D       0.93      0.90      0.91       449\n",
            "           T       0.65      0.47      0.55        36\n",
            "           L       0.97      0.89      0.93       129\n",
            "           P       0.89      0.92      0.90       616\n",
            "           ^       0.83      0.64      0.72       495\n",
            "\n",
            "   micro avg       0.87      0.87      0.87      8152\n",
            "   macro avg       0.65      0.62      0.63      8152\n",
            "weighted avg       0.87      0.87      0.87      8152\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([0.7006891369819641,\n",
              "  0.45925208926200867,\n",
              "  0.31481999158859253,\n",
              "  0.4070170223712921,\n",
              "  0.29012858867645264,\n",
              "  0.45144855976104736,\n",
              "  0.5042242407798767,\n",
              "  0.3511987626552582,\n",
              "  0.4751989543437958,\n",
              "  0.23687487840652466,\n",
              "  0.6624816060066223,\n",
              "  0.6872777342796326,\n",
              "  0.47881096601486206,\n",
              "  0.30139902234077454,\n",
              "  0.17587785422801971,\n",
              "  0.6536751389503479,\n",
              "  0.615035355091095,\n",
              "  1.0061370134353638,\n",
              "  0.18497812747955322,\n",
              "  0.22598868608474731,\n",
              "  0.5676613450050354,\n",
              "  0.7599863409996033,\n",
              "  0.6863417625427246,\n",
              "  0.23041629791259766,\n",
              "  0.206008642911911,\n",
              "  0.8092441558837891,\n",
              "  0.3676867187023163,\n",
              "  0.5644270181655884,\n",
              "  0.65610271692276,\n",
              "  0.8222359418869019,\n",
              "  0.3331598937511444,\n",
              "  0.8454391956329346,\n",
              "  0.5425511598587036,\n",
              "  0.6519476175308228,\n",
              "  0.7522041201591492,\n",
              "  0.635339617729187,\n",
              "  0.40970155596733093,\n",
              "  0.6522890329360962,\n",
              "  0.34531134366989136,\n",
              "  0.7003124952316284,\n",
              "  0.9348519444465637,\n",
              "  0.35227182507514954,\n",
              "  0.5842916965484619,\n",
              "  0.4468477964401245,\n",
              "  0.5370994210243225,\n",
              "  0.6752871870994568,\n",
              "  0.5304470658302307,\n",
              "  0.4848797619342804,\n",
              "  0.5803802013397217,\n",
              "  0.485309898853302,\n",
              "  0.6619414687156677,\n",
              "  0.22985929250717163,\n",
              "  0.5642890334129333,\n",
              "  0.44340813159942627,\n",
              "  0.4769543409347534,\n",
              "  0.5848719477653503,\n",
              "  0.7325208187103271,\n",
              "  0.3852422833442688,\n",
              "  0.42614850401878357,\n",
              "  0.3001493811607361,\n",
              "  0.9513884782791138,\n",
              "  1.0672593116760254,\n",
              "  0.6230599284172058,\n",
              "  0.386293888092041,\n",
              "  0.48052749037742615,\n",
              "  0.7294942736625671,\n",
              "  0.9581189751625061,\n",
              "  0.22946684062480927,\n",
              "  0.34014222025871277,\n",
              "  0.37070778012275696,\n",
              "  0.6023781895637512,\n",
              "  0.14583900570869446,\n",
              "  0.6902962923049927,\n",
              "  0.2376151829957962,\n",
              "  0.9774487018585205,\n",
              "  0.3444553315639496,\n",
              "  0.6384506821632385,\n",
              "  0.30182740092277527,\n",
              "  0.5014670491218567,\n",
              "  0.3721044063568115,\n",
              "  0.7128334045410156,\n",
              "  0.3277343511581421,\n",
              "  0.44650930166244507,\n",
              "  0.19988787174224854,\n",
              "  0.4177130162715912,\n",
              "  0.34076938033103943,\n",
              "  0.6661677360534668,\n",
              "  0.49135667085647583,\n",
              "  0.576764702796936,\n",
              "  0.6043291091918945,\n",
              "  0.36072856187820435,\n",
              "  0.6178639531135559,\n",
              "  0.35891634225845337,\n",
              "  0.8150481581687927,\n",
              "  0.6221047639846802,\n",
              "  0.543250322341919,\n",
              "  0.5288152694702148,\n",
              "  0.8268663883209229,\n",
              "  0.42219942808151245,\n",
              "  0.2454366832971573,\n",
              "  0.5153092741966248,\n",
              "  0.6582012176513672,\n",
              "  0.4863032400608063,\n",
              "  0.7777831554412842,\n",
              "  0.33577531576156616,\n",
              "  0.6312017440795898,\n",
              "  0.5966749787330627,\n",
              "  0.3990117013454437,\n",
              "  0.40635037422180176,\n",
              "  0.27595287561416626,\n",
              "  0.77668696641922,\n",
              "  0.38952744007110596,\n",
              "  0.3504898250102997,\n",
              "  0.5615119338035583,\n",
              "  0.5345914959907532,\n",
              "  0.7671619057655334,\n",
              "  0.8501495718955994,\n",
              "  0.4700511395931244,\n",
              "  0.29533588886260986,\n",
              "  0.44270962476730347,\n",
              "  0.24522511661052704,\n",
              "  0.17033889889717102,\n",
              "  0.5730345249176025,\n",
              "  0.3266606628894806,\n",
              "  0.4266294538974762],\n",
              " [0.7006891369819641,\n",
              "  0.5799706131219864,\n",
              "  0.4915870726108551,\n",
              "  0.47044456005096436,\n",
              "  0.434381365776062,\n",
              "  0.4372258981068929,\n",
              "  0.44679708991731915,\n",
              "  0.43484729900956154,\n",
              "  0.4393308162689209,\n",
              "  0.41908522248268126,\n",
              "  0.4412121664394032,\n",
              "  0.461717630426089,\n",
              "  0.46303250239445615,\n",
              "  0.4514872538191932,\n",
              "  0.4331132938464483,\n",
              "  0.44689840916544205,\n",
              "  0.45678881774930397,\n",
              "  0.48730816195408505,\n",
              "  0.4713960548764781,\n",
              "  0.45912568643689156,\n",
              "  0.4642940511306127,\n",
              "  0.47773460976102133,\n",
              "  0.48680448596892145,\n",
              "  0.476121644799908,\n",
              "  0.46531712472438813,\n",
              "  0.47854508746128815,\n",
              "  0.4744392219516966,\n",
              "  0.4776530718164784,\n",
              "  0.4838065078546261,\n",
              "  0.49508748898903526,\n",
              "  0.48986401817490977,\n",
              "  0.500975742470473,\n",
              "  0.5022356036034498,\n",
              "  0.5066388981307255,\n",
              "  0.5136550473315376,\n",
              "  0.5170351742870278,\n",
              "  0.5141342656837928,\n",
              "  0.5177699174535902,\n",
              "  0.5133479027411877,\n",
              "  0.5180220175534487,\n",
              "  0.5281886011362076,\n",
              "  0.5240001064680871,\n",
              "  0.5254022364699563,\n",
              "  0.5236169082874601,\n",
              "  0.5239165196816126,\n",
              "  0.5272071863646092,\n",
              "  0.5272761199702607,\n",
              "  0.5263928625111778,\n",
              "  0.5274946449362501,\n",
              "  0.5266509500145912,\n",
              "  0.5293037052832398,\n",
              "  0.5235451588837,\n",
              "  0.5243139112333082,\n",
              "  0.522815656054903,\n",
              "  0.5219818139618093,\n",
              "  0.523104852065444,\n",
              "  0.5267788163925472,\n",
              "  0.5243385313399906,\n",
              "  0.5226742935887837,\n",
              "  0.5189655450483163,\n",
              "  0.5260544455930835,\n",
              "  0.5347835563363567,\n",
              "  0.5361847685916082,\n",
              "  0.5338427235838025,\n",
              "  0.5330224892267813,\n",
              "  0.5359993344455053,\n",
              "  0.5422996275905352,\n",
              "  0.5376991454292745,\n",
              "  0.5348360015862229,\n",
              "  0.5324913127081734,\n",
              "  0.5334756349174069,\n",
              "  0.5280917928450637,\n",
              "  0.5303137722897202,\n",
              "  0.5263583859479104,\n",
              "  0.5323729234933852,\n",
              "  0.5299003235995768,\n",
              "  0.531310068515988,\n",
              "  0.5283679830340238,\n",
              "  0.5280274648832368,\n",
              "  0.5260784266516565,\n",
              "  0.5283840436626362,\n",
              "  0.5259370961930693,\n",
              "  0.5249801348131823,\n",
              "  0.5211099888242426,\n",
              "  0.5198935538530349,\n",
              "  0.517810714626035,\n",
              "  0.5195159677458906,\n",
              "  0.5191959757357836,\n",
              "  0.5198428153656842,\n",
              "  0.5207815519637532,\n",
              "  0.5190227278968791,\n",
              "  0.5200970890405386,\n",
              "  0.5183639627310539,\n",
              "  0.5215201775761362,\n",
              "  0.5225789626962262,\n",
              "  0.5227942893592021,\n",
              "  0.5228563613191095,\n",
              "  0.525958504451781,\n",
              "  0.5249104329732935,\n",
              "  0.5221156954765321,\n",
              "  0.5220483051668301,\n",
              "  0.5233831376421687,\n",
              "  0.5230231386365244,\n",
              "  0.525472754182724,\n",
              "  0.5236661119120463,\n",
              "  0.5246805990079665,\n",
              "  0.5253534436782945,\n",
              "  0.5241836127307681,\n",
              "  0.5231025738453647,\n",
              "  0.5208557584068993,\n",
              "  0.523160544064668,\n",
              "  0.5219673913504398,\n",
              "  0.5204498907633589,\n",
              "  0.5208100841233606,\n",
              "  0.5209299224874249,\n",
              "  0.523052611998443,\n",
              "  0.5258483125103846,\n",
              "  0.5253754551127806,\n",
              "  0.5234423495140397,\n",
              "  0.5227695768078169,\n",
              "  0.5204758209384178,\n",
              "  0.517605846167588,\n",
              "  0.5180564858322222,\n",
              "  0.5165129711310711,\n",
              "  0.5157939029932024])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oKPYRqEYsgbr"
      },
      "source": [
        "# References\n",
        "* Part-of-Speech Tagging from 97% to 100%:\n",
        "Is It Time for Some Linguistics? https://nlp.stanford.edu/pubs/CICLing2011-manning-tagging.pdf\n",
        "* Twitter Word Embeddings: https://fredericgodin.com/research/twitter-word-embeddings/"
      ]
    }
  ]
}