Longformer_for_sequence_classification_imdb.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Longformer for sequence classification imdb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOJCnTmu/08iwhTVHuuqOaY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "54f779640226475386428d13cb6d5dd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6f78b7dabc034152b7cd1645e7d06408",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_75320e5bb6354b44884a9a9efb169355",
              "IPY_MODEL_b350b7706a7a426295f4c9c86ea3a878"
            ]
          }
        },
        "6f78b7dabc034152b7cd1645e7d06408": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "75320e5bb6354b44884a9a9efb169355": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8e2443e0eab24a678213710ee5b2d626",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 898823,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 898823,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_582d2ebf9ea5479d88e4146a4c70446c"
          }
        },
        "b350b7706a7a426295f4c9c86ea3a878": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c024a218d2b4457896759e18dfad573c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 899k/899k [00:00&lt;00:00, 907kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0c6e5f088029442180b658f85e09939d"
          }
        },
        "8e2443e0eab24a678213710ee5b2d626": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "582d2ebf9ea5479d88e4146a4c70446c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c024a218d2b4457896759e18dfad573c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0c6e5f088029442180b658f85e09939d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd7301340a5e4380b6fac41b00517180": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_31da8d58c47f4f7a83c3c94bcbd42301",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_91f2ec6602da48b38f3000f72370dd5f",
              "IPY_MODEL_a0972cbabf734fd2bb6aed930569e23a"
            ]
          }
        },
        "31da8d58c47f4f7a83c3c94bcbd42301": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "91f2ec6602da48b38f3000f72370dd5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_128e22f4255d40fd89ed18fdf88c2fd6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b7ff97fd34084871844cb7392e1955d9"
          }
        },
        "a0972cbabf734fd2bb6aed930569e23a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cfeb164449a34e97bf67e423bfd05ea5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:02&lt;00:00, 227kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_64c5f7c5444d440da8c2cec7e8f0eb83"
          }
        },
        "128e22f4255d40fd89ed18fdf88c2fd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b7ff97fd34084871844cb7392e1955d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cfeb164449a34e97bf67e423bfd05ea5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "64c5f7c5444d440da8c2cec7e8f0eb83": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ad3e2114d5a64c47986a361fcf203eb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_21cc03b1aa474da69e734be8b8ef645d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e9f8b50edb9246d69d80c6daa6ec6c41",
              "IPY_MODEL_7492cd0423744790845001c4331324f9"
            ]
          }
        },
        "21cc03b1aa474da69e734be8b8ef645d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e9f8b50edb9246d69d80c6daa6ec6c41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9b0f29a8213046d59b3a27a6993c054e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4520,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4520,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7938a487c8ff4b95b900a56fb8c67b11"
          }
        },
        "7492cd0423744790845001c4331324f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c08a2ef4dcdb4b74931ae5ab47dde986",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4.52k/4.52k [00:00&lt;00:00, 13.3kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5401dfd74b8c466ea98ff2740697abab"
          }
        },
        "9b0f29a8213046d59b3a27a6993c054e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7938a487c8ff4b95b900a56fb8c67b11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c08a2ef4dcdb4b74931ae5ab47dde986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5401dfd74b8c466ea98ff2740697abab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7191431ee43e4ab4b2834e1847a2b8ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8c5e448abdfe4a46bbeb8eccc8c750a0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9541e827b09b40f79ec1d881a7121edc",
              "IPY_MODEL_cd250055fa044db5bb89ab6ee09b6be9"
            ]
          }
        },
        "8c5e448abdfe4a46bbeb8eccc8c750a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9541e827b09b40f79ec1d881a7121edc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0822240401a148a2924c52b0600856f1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2071,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2071,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2d959bb763334cb7b599adf60a2dd945"
          }
        },
        "cd250055fa044db5bb89ab6ee09b6be9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e4348c79b6c7476681f7250a945538f9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.07k/2.07k [00:00&lt;00:00, 5.84kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f32cfe3b58894cbb95f3c41d95c8b7ff"
          }
        },
        "0822240401a148a2924c52b0600856f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2d959bb763334cb7b599adf60a2dd945": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e4348c79b6c7476681f7250a945538f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f32cfe3b58894cbb95f3c41d95c8b7ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "78088a2989e9453482f2783ad33c8cee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6db87ae154cf4a17910afbb5e6b6a015",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dd8d2369d13c482694711e96e15651c6",
              "IPY_MODEL_1c43281eb9864b07a3e48a9633cb290d"
            ]
          }
        },
        "6db87ae154cf4a17910afbb5e6b6a015": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd8d2369d13c482694711e96e15651c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cc4ad3837e554b338cf121b22810f97a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 84125825,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 84125825,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1fa018f134914f8ea4ec07f514983ea4"
          }
        },
        "1c43281eb9864b07a3e48a9633cb290d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7d87675484bd4347a7e6492d938163ac",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 84.1M/84.1M [00:08&lt;00:00, 9.56MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5fb273e7defe458bbf9f929a8f5f6891"
          }
        },
        "cc4ad3837e554b338cf121b22810f97a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1fa018f134914f8ea4ec07f514983ea4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d87675484bd4347a7e6492d938163ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5fb273e7defe458bbf9f929a8f5f6891": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "645165214fb448c6b6422878db477232": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_26b92b45563740c6befd839f74b9d57a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_11ccbf92af304e6c8b9e64fdd9e1f6f8",
              "IPY_MODEL_441df3da82eb416a9a32cb4da6d3e5fe"
            ]
          }
        },
        "26b92b45563740c6befd839f74b9d57a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "11ccbf92af304e6c8b9e64fdd9e1f6f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a4e392b0ad154d2989fe569b45c5b548",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a741721aa014ae6a380cfba7da0bdba"
          }
        },
        "441df3da82eb416a9a32cb4da6d3e5fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3896b4ab4f084ac3aa5e56019b05ceb3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 25000/0 [00:01&lt;00:00, 21800.90 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e4d0fe556104485b5776f718c83ad13"
          }
        },
        "a4e392b0ad154d2989fe569b45c5b548": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a741721aa014ae6a380cfba7da0bdba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3896b4ab4f084ac3aa5e56019b05ceb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e4d0fe556104485b5776f718c83ad13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "321256ca10d24cbea9e20798e61fbdd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_35617f8ff96645d1a9adae7b445e867a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bf15881279da4467b3b1e1a731c98f49",
              "IPY_MODEL_2a2774cf64c24f3d96152f58242e3ec2"
            ]
          }
        },
        "35617f8ff96645d1a9adae7b445e867a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bf15881279da4467b3b1e1a731c98f49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b970fe55169a45a8a2aa2d80ff0d00bf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_973e86826e3440c9b7a1222528e05195"
          }
        },
        "2a2774cf64c24f3d96152f58242e3ec2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_89ae6cb8b1124423b146e15b53906346",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 25000/0 [00:01&lt;00:00, 22969.25 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d8e7aec6106a48038a4f5fc7a56dd18b"
          }
        },
        "b970fe55169a45a8a2aa2d80ff0d00bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "973e86826e3440c9b7a1222528e05195": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "89ae6cb8b1124423b146e15b53906346": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d8e7aec6106a48038a4f5fc7a56dd18b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "244f29b3f5d543909e30ca76d9dfea84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_af4da6def7e34a7786a0f3c9814809d7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a4aa54d6777846519900ffd27f6dccdb",
              "IPY_MODEL_7ed216c33476490ab441fab54861845c"
            ]
          }
        },
        "af4da6def7e34a7786a0f3c9814809d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a4aa54d6777846519900ffd27f6dccdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_33f80f9d3d1b4c16b8b658916705cf64",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ba9c1b01b674b3cb8e1f74e9fc53d14"
          }
        },
        "7ed216c33476490ab441fab54861845c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_af2a9bd088af485facf006bccf04d48d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 50000/0 [00:02&lt;00:00, 22977.13 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0b995b2f12354978a17cf1a2f434e025"
          }
        },
        "33f80f9d3d1b4c16b8b658916705cf64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ba9c1b01b674b3cb8e1f74e9fc53d14": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af2a9bd088af485facf006bccf04d48d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0b995b2f12354978a17cf1a2f434e025": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "97139df82c7d45a29baf6f0e6f5f789c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_05a60c770f84443d8dc98f1f18d77408",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_10aae66470f74182a0c62b9f65f34947",
              "IPY_MODEL_21751f26145d4c0aa87181934585ce78"
            ]
          }
        },
        "05a60c770f84443d8dc98f1f18d77408": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10aae66470f74182a0c62b9f65f34947": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5ce1ca7061074db2aac4b53628e022d1",
            "_dom_classes": [],
            "description": "Epoch:   0%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 0,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ac2ca3d91c9442a8912c8363626783fb"
          }
        },
        "21751f26145d4c0aa87181934585ce78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8ea846c7b9f44087af1f7bb6022a0f84",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0/3 [00:06&lt;?, ?it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9af23bd3578b4089bd639fb1d5b36b1d"
          }
        },
        "5ce1ca7061074db2aac4b53628e022d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ac2ca3d91c9442a8912c8363626783fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8ea846c7b9f44087af1f7bb6022a0f84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9af23bd3578b4089bd639fb1d5b36b1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf2f4112938e4307be65719cb1f91416": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d1494b6ba2e54e93a29e0aaaaa37f14d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5d3b1dfaeba24cf480f38fb5a48a9106",
              "IPY_MODEL_dcdd52ac8bc141d9ac945e5ed0c01fff"
            ]
          }
        },
        "d1494b6ba2e54e93a29e0aaaaa37f14d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d3b1dfaeba24cf480f38fb5a48a9106": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a529f5d01039461a86fd13d50de4ca0e",
            "_dom_classes": [],
            "description": "Iteration:   0%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 391,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 0,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d7c226773aa4d1598b594d3d9de008f"
          }
        },
        "dcdd52ac8bc141d9ac945e5ed0c01fff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e8b84925accd4500b0296321367393a3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0/391 [00:06&lt;?, ?it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ea9d7244c4144ba986b453d87780e217"
          }
        },
        "a529f5d01039461a86fd13d50de4ca0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d7c226773aa4d1598b594d3d9de008f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e8b84925accd4500b0296321367393a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ea9d7244c4144ba986b453d87780e217": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/manishiitg/ML_Experiments/blob/master/Longformer_for_sequence_classification_imdb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FZZLsxQGztn3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        },
        "outputId": "0b6db87b-255c-42e2-fc2c-6b50a276d9a5"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Jun 15 16:05:14 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 450.36.06    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   33C    P0    32W / 250W |  13051MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GV9SbRoM58fq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "df844163-8f62-4eb5-aff4-1c69213fbb04"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers.git\n",
        "!pip install -U ./transformers\n",
        "!pip install git+https://github.com/huggingface/nlp.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 33, done.\u001b[K\n",
            "remote: Counting objects: 100% (33/33), done.\u001b[K\n",
            "remote: Compressing objects: 100% (24/24), done.\u001b[K\n",
            "remote: Total 29311 (delta 11), reused 16 (delta 3), pack-reused 29278\u001b[K\n",
            "Receiving objects: 100% (29311/29311), 26.63 MiB | 30.81 MiB/s, done.\n",
            "Resolving deltas: 100% (20273/20273), done.\n",
            "Processing ./transformers\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (1.18.5)\n",
            "Collecting tokenizers==0.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/e5/a26eb4716523808bb0a799fcfdceb6ebf77a18169d9591b2f46a9adb87d9/tokenizers-0.7.0-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8MB 4.5MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (20.4)\n",
            "Requirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (3.0.12)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (4.41.1)\n",
            "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (2019.12.20)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 28.8MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 23.6MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: dataclasses in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (0.7)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==2.11.0) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==2.11.0) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.11.0) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.11.0) (2020.4.5.1)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.11.0) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.11.0) (2.9)\n",
            "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.11.0) (7.1.2)\n",
            "Requirement already satisfied, skipping upgrade: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.11.0) (0.15.1)\n",
            "Building wheels for collected packages: transformers, sacremoses\n",
            "  Building wheel for transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-2.11.0-cp36-none-any.whl size=688581 sha256=71571895398c11c76cd112f6112098e04a5f316a6f4bff2b407432f9314525ce\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-h9vu2ami/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=ceffbe8a13b526cae5c5d60659c9787595cd72a5bab4c3704560bab6b7019781\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built transformers sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.7.0 transformers-2.11.0\n",
            "Collecting git+https://github.com/huggingface/nlp.git\n",
            "  Cloning https://github.com/huggingface/nlp.git to /tmp/pip-req-build-opdp2znq\n",
            "  Running command git clone -q https://github.com/huggingface/nlp.git /tmp/pip-req-build-opdp2znq\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from nlp==0.2.1) (1.18.5)\n",
            "Collecting pyarrow>=0.16.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ba/3f/6cac1714fff444664603f92cb9fbe91c7ae25375880158b9e9691c4584c8/pyarrow-0.17.1-cp36-cp36m-manylinux2014_x86_64.whl (63.8MB)\n",
            "\u001b[K     |████████████████████████████████| 63.8MB 44kB/s \n",
            "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from nlp==0.2.1) (0.3.1.1)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.6/dist-packages (from nlp==0.2.1) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from nlp==0.2.1) (4.41.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from nlp==0.2.1) (3.0.12)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from nlp==0.2.1) (0.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->nlp==0.2.1) (2.9)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->nlp==0.2.1) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->nlp==0.2.1) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->nlp==0.2.1) (2020.4.5.1)\n",
            "Building wheels for collected packages: nlp\n",
            "  Building wheel for nlp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nlp: filename=nlp-0.2.1-cp36-none-any.whl size=99932 sha256=a3fba562afb2f8c048a9ec712d79e92a5052fb753bfc8415ef707439c80bab26\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-kzjierq8/wheels/a2/98/bd/15f115cb85f1a049a96b915aa14466d3b51a023cd597e08e0d\n",
            "Successfully built nlp\n",
            "Installing collected packages: pyarrow, nlp\n",
            "  Found existing installation: pyarrow 0.14.1\n",
            "    Uninstalling pyarrow-0.14.1:\n",
            "      Successfully uninstalled pyarrow-0.14.1\n",
            "Successfully installed nlp-0.2.1 pyarrow-0.17.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "pyarrow"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bo_m2vny5-LU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from transformers import LongformerTokenizerFast"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Wm0TvLC6CFK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "54f779640226475386428d13cb6d5dd4",
            "6f78b7dabc034152b7cd1645e7d06408",
            "75320e5bb6354b44884a9a9efb169355",
            "b350b7706a7a426295f4c9c86ea3a878",
            "8e2443e0eab24a678213710ee5b2d626",
            "582d2ebf9ea5479d88e4146a4c70446c",
            "c024a218d2b4457896759e18dfad573c",
            "0c6e5f088029442180b658f85e09939d",
            "dd7301340a5e4380b6fac41b00517180",
            "31da8d58c47f4f7a83c3c94bcbd42301",
            "91f2ec6602da48b38f3000f72370dd5f",
            "a0972cbabf734fd2bb6aed930569e23a",
            "128e22f4255d40fd89ed18fdf88c2fd6",
            "b7ff97fd34084871844cb7392e1955d9",
            "cfeb164449a34e97bf67e423bfd05ea5",
            "64c5f7c5444d440da8c2cec7e8f0eb83"
          ]
        },
        "outputId": "7d12dc27-4d6a-435e-f926-fa26a8d98fcc"
      },
      "source": [
        "tokenizer = LongformerTokenizerFast.from_pretrained('allenai/longformer-base-4096')\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "54f779640226475386428d13cb6d5dd4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dd7301340a5e4380b6fac41b00517180",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T6YC5KXS6CQn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 240,
          "referenced_widgets": [
            "ad3e2114d5a64c47986a361fcf203eb1",
            "21cc03b1aa474da69e734be8b8ef645d",
            "e9f8b50edb9246d69d80c6daa6ec6c41",
            "7492cd0423744790845001c4331324f9",
            "9b0f29a8213046d59b3a27a6993c054e",
            "7938a487c8ff4b95b900a56fb8c67b11",
            "c08a2ef4dcdb4b74931ae5ab47dde986",
            "5401dfd74b8c466ea98ff2740697abab",
            "7191431ee43e4ab4b2834e1847a2b8ae",
            "8c5e448abdfe4a46bbeb8eccc8c750a0",
            "9541e827b09b40f79ec1d881a7121edc",
            "cd250055fa044db5bb89ab6ee09b6be9",
            "0822240401a148a2924c52b0600856f1",
            "2d959bb763334cb7b599adf60a2dd945",
            "e4348c79b6c7476681f7250a945538f9",
            "f32cfe3b58894cbb95f3c41d95c8b7ff",
            "78088a2989e9453482f2783ad33c8cee",
            "6db87ae154cf4a17910afbb5e6b6a015",
            "dd8d2369d13c482694711e96e15651c6",
            "1c43281eb9864b07a3e48a9633cb290d",
            "cc4ad3837e554b338cf121b22810f97a",
            "1fa018f134914f8ea4ec07f514983ea4",
            "7d87675484bd4347a7e6492d938163ac",
            "5fb273e7defe458bbf9f929a8f5f6891",
            "645165214fb448c6b6422878db477232",
            "26b92b45563740c6befd839f74b9d57a",
            "11ccbf92af304e6c8b9e64fdd9e1f6f8",
            "441df3da82eb416a9a32cb4da6d3e5fe",
            "a4e392b0ad154d2989fe569b45c5b548",
            "3a741721aa014ae6a380cfba7da0bdba",
            "3896b4ab4f084ac3aa5e56019b05ceb3",
            "7e4d0fe556104485b5776f718c83ad13",
            "321256ca10d24cbea9e20798e61fbdd5",
            "35617f8ff96645d1a9adae7b445e867a",
            "bf15881279da4467b3b1e1a731c98f49",
            "2a2774cf64c24f3d96152f58242e3ec2",
            "b970fe55169a45a8a2aa2d80ff0d00bf",
            "973e86826e3440c9b7a1222528e05195",
            "89ae6cb8b1124423b146e15b53906346",
            "d8e7aec6106a48038a4f5fc7a56dd18b",
            "244f29b3f5d543909e30ca76d9dfea84",
            "af4da6def7e34a7786a0f3c9814809d7",
            "a4aa54d6777846519900ffd27f6dccdb",
            "7ed216c33476490ab441fab54861845c",
            "33f80f9d3d1b4c16b8b658916705cf64",
            "3ba9c1b01b674b3cb8e1f74e9fc53d14",
            "af2a9bd088af485facf006bccf04d48d",
            "0b995b2f12354978a17cf1a2f434e025"
          ]
        },
        "outputId": "55e0158a-4439-4c88-e429-2692644d05d3"
      },
      "source": [
        "import nlp\n",
        "\n",
        "dataset = nlp.load_dataset('imdb', 'plain_text')\n",
        "print(dataset)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ad3e2114d5a64c47986a361fcf203eb1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=4520.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7191431ee43e4ab4b2834e1847a2b8ae",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2071.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Downloading and preparing dataset imdb/plain_text (download: 80.23 MiB, generated: 127.06 MiB, total: 207.28 MiB) to /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "78088a2989e9453482f2783ad33c8cee",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=84125825.0, style=ProgressStyle(descrip…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "645165214fb448c6b6422878db477232",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "321256ca10d24cbea9e20798e61fbdd5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "244f29b3f5d543909e30ca76d9dfea84",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rDataset imdb downloaded and prepared to /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0. Subsequent calls will reuse this data.\n",
            "{'train': Dataset(schema: {'text': 'string', 'label': 'int64'}, num_rows: 25000), 'test': Dataset(schema: {'text': 'string', 'label': 'int64'}, num_rows: 25000), 'unsupervised': Dataset(schema: {'text': 'string', 'label': 'int64'}, num_rows: 50000)}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VDVcjxIX_bIK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "d86c4f9a-b126-4a6a-ec5d-78878362fad2"
      },
      "source": [
        "dataset[\"train\"][10000]"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'label': 1,\n",
              " 'text': \"I really liked Tom Barman's AWTWB. You just have to let it come over you and enjoy it while it lasts, and don't expect anything. It's like sitting on a café-terrace with a beer, in the summer sun, and watching the people go by. It definitely won't keep you pondering afterwards, that's true, but that's not a prerequisite for a good film. It's just the experience during the movie that's great.<br /><br />I felt there were a few strands that could have been worked out a little more, but being a Lynch fan I don't care that much anymore :)<br /><br />And I *loved* the style, or flair of this movie. It's slick, but fresh, and the soundtrack is a beauty. Any music-lover will get his kicks out of AWTWB, I can assure you.<br /><br />I'll give it 8 out 10.<br /><br />(music-wise 10 out of 10)\"}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LOe1Lg5ZC147",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Tokenize our training dataset\n",
        "def convert_to_features(example):\n",
        "    # Tokenize contexts and questions (as pairs of inputs)\n",
        "    encodings = tokenizer.encode_plus(example[\"text\"], pad_to_max_length=True, max_length=512)    \n",
        "    encodings.update({'label': example[\"label\"]})\n",
        "\n",
        "    return encodings\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VC_gOOeXH9pK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "c7be6f34-72da-4bce-ffa4-678ccd670455"
      },
      "source": [
        "train_dataset  = nlp.load_dataset('imdb', 'plain_text', split=nlp.Split.TRAIN)\n",
        "valid_dataset = nlp.load_dataset('imdb', 'plain_text', split=nlp.Split.TEST)\n",
        "\n",
        "\n",
        "train_dataset = train_dataset.map(convert_to_features)\n",
        "valid_dataset = valid_dataset.map(convert_to_features, load_from_cache_file=False)\n",
        "\n",
        "\n",
        "# set the tensor type and the columns which the dataset should return\n",
        "columns = ['input_ids', 'attention_mask','label']\n",
        "train_dataset.set_format(type='torch', columns=columns)\n",
        "valid_dataset.set_format(type='torch', columns=columns)\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "25000it [00:20, 1241.45it/s]\n",
            "25000it [00:19, 1258.82it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k7cv7QFjIPcQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9a97d3c7-f3fc-49a4-de78-f74d2923b336"
      },
      "source": [
        "len(train_dataset), len(valid_dataset)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(25000, 25000)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_Y-5lUPQOoH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.save(train_dataset, 'train_data.pt')\n",
        "torch.save(valid_dataset, 'valid_data.pt')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XHqGVJe8QP2B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import dataclasses\n",
        "import logging\n",
        "import os\n",
        "import sys\n",
        "from dataclasses import dataclass, field\n",
        "from typing import Dict, List, Optional\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "\n",
        "from transformers import LongformerForSequenceClassification, LongformerTokenizerFast, EvalPrediction\n",
        "from transformers import (\n",
        "    HfArgumentParser,\n",
        "    DataCollator,\n",
        "    Trainer,\n",
        "    TrainingArguments,\n",
        "    set_seed,\n",
        ")\n",
        "\n",
        "\n",
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "# @dataclass\n",
        "class DummyDataCollator(DataCollator):\n",
        "    def collate_batch(self, batch: List) -> Dict[str, torch.Tensor]:\n",
        "        \"\"\"\n",
        "        Take a list of samples from a Dataset and collate them into a batch.\n",
        "        Returns:\n",
        "            A dictionary of tensors\n",
        "        \"\"\"\n",
        "        input_ids = torch.stack([example['input_ids'] for example in batch])\n",
        "        attention_mask = torch.stack([example['attention_mask'] for example in batch])\n",
        "        label = torch.stack([example['label'] for example in batch])\n",
        "\n",
        "        return {\n",
        "            'input_ids': input_ids, \n",
        "            'labels': label, \n",
        "            'attention_mask': attention_mask\n",
        "        }\n",
        "\n",
        "\n",
        "@dataclass\n",
        "class ModelArguments:\n",
        "    \"\"\"\n",
        "    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.\n",
        "    \"\"\"\n",
        "\n",
        "    model_name_or_path: str = field(\n",
        "        metadata={\"help\": \"Path to pretrained model or model identifier from huggingface.co/models\"}\n",
        "    )\n",
        "    tokenizer_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n",
        "    )\n",
        "    cache_dir: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Where do you want to store the pretrained models downloaded from s3\"}\n",
        "    )\n",
        "\n",
        "@dataclass\n",
        "class DataTrainingArguments:\n",
        "    \"\"\"\n",
        "    Arguments pertaining to what data we are going to input our model for training and eval.\n",
        "    \"\"\"\n",
        "    train_file_path: Optional[str] = field(\n",
        "        default='train_data.pt',\n",
        "        metadata={\"help\": \"Path for cached train dataset\"},\n",
        "    )\n",
        "    valid_file_path: Optional[str] = field(\n",
        "        default='valid_data.pt',\n",
        "        metadata={\"help\": \"Path for cached valid dataset\"},\n",
        "    )\n",
        "    max_len: Optional[int] = field(\n",
        "        default=512,\n",
        "        metadata={\"help\": \"Max input length for the source text\"},\n",
        "    )\n",
        "\n",
        "\n",
        "def main():\n",
        "    # See all possible arguments in src/transformers/training_args.py\n",
        "    # or by passing the --help flag to this script.\n",
        "    # We now keep distinct sets of args, for a cleaner separation of concerns.\n",
        "\n",
        "    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n",
        "\n",
        "    # we will load the arguments from a json file, \n",
        "    # make sure you save the arguments in at ./args.json\n",
        "    model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath('args.json'))\n",
        "\n",
        "    if (\n",
        "        os.path.exists(training_args.output_dir)\n",
        "        and os.listdir(training_args.output_dir)\n",
        "        and training_args.do_train\n",
        "        and not training_args.overwrite_output_dir\n",
        "    ):\n",
        "        raise ValueError(\n",
        "            f\"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome.\"\n",
        "        )\n",
        "\n",
        "    # Setup logging\n",
        "    logging.basicConfig(\n",
        "        format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
        "        datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,\n",
        "    )\n",
        "    logger.warning(\n",
        "        \"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s\",\n",
        "        training_args.local_rank,\n",
        "        training_args.device,\n",
        "        training_args.n_gpu,\n",
        "        bool(training_args.local_rank != -1),\n",
        "        training_args.fp16,\n",
        "    )\n",
        "    logger.info(\"Training/evaluation parameters %s\", training_args)\n",
        "\n",
        "    # Set seed\n",
        "    set_seed(training_args.seed)\n",
        "\n",
        "    # Load pretrained model and tokenizer\n",
        "    #\n",
        "    # Distributed training:\n",
        "    # The .from_pretrained methods guarantee that only one local process can concurrently\n",
        "    # download model & vocab.\n",
        "\n",
        "    tokenizer = LongformerTokenizerFast.from_pretrained(\n",
        "        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n",
        "        cache_dir=model_args.cache_dir,\n",
        "    )\n",
        "    model = LongformerForSequenceClassification.from_pretrained(\n",
        "        model_args.model_name_or_path,\n",
        "        cache_dir=model_args.cache_dir,\n",
        "    )\n",
        "\n",
        "    # Get datasets\n",
        "    # print('loading data')\n",
        "    # train_dataset  = torch.load(data_args.train_file_path)\n",
        "    # valid_dataset = torch.load(data_args.valid_file_path)\n",
        "    # print('loading done')\n",
        "\n",
        "    # Initialize our Trainer\n",
        "    trainer = Trainer(\n",
        "        model=model,\n",
        "        args=training_args,\n",
        "        train_dataset=train_dataset,\n",
        "        eval_dataset=valid_dataset,\n",
        "        data_collator=DummyDataCollator(),\n",
        "        prediction_loss_only=True,\n",
        "    )\n",
        "\n",
        "    # Training\n",
        "    if training_args.do_train:\n",
        "        trainer.train(\n",
        "            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None\n",
        "        )\n",
        "        trainer.save_model()\n",
        "        # For convenience, we also re-save the tokenizer to the same directory,\n",
        "        # so that you can share your model easily on huggingface.co/models =)\n",
        "        if trainer.is_world_master():\n",
        "            tokenizer.save_pretrained(training_args.output_dir)\n",
        "\n",
        "    # Evaluation\n",
        "    results = {}\n",
        "    if training_args.do_eval and training_args.local_rank in [-1, 0]:\n",
        "        logger.info(\"*** Evaluate ***\")\n",
        "\n",
        "        eval_output = trainer.evaluate()\n",
        "\n",
        "        output_eval_file = os.path.join(training_args.output_dir, \"eval_results.txt\")\n",
        "        with open(output_eval_file, \"w\") as writer:\n",
        "            logger.info(\"***** Eval results *****\")\n",
        "            for key in sorted(eval_output.keys()):\n",
        "                logger.info(\"  %s = %s\", key, str(eval_output[key]))\n",
        "                writer.write(\"%s = %s\\n\" % (key, str(eval_output[key])))\n",
        "    \n",
        "        results.update(eval_output)\n",
        "    \n",
        "    return results\n",
        "\n",
        "\n",
        "def _mp_fn(index):\n",
        "    # For xla_spawn (TPUs)\n",
        "    main()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n0oWTa4VQigB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import json\n",
        "args_dict = {\n",
        "  \"n_gpu\": 1,\n",
        "  \"model_name_or_path\": 'allenai/longformer-base-4096',\n",
        "  \"max_len\": 128 ,\n",
        "  \"output_dir\": './models',\n",
        "  \"overwrite_output_dir\": True,\n",
        "  \"per_gpu_train_batch_size\": 8,\n",
        "  \"per_gpu_eval_batch_size\": 8,\n",
        "  \"gradient_accumulation_steps\": 16,\n",
        "  \"learning_rate\": 1e-4,\n",
        "  \"num_train_epochs\": 3,\n",
        "  \"do_train\": True,\n",
        "  \"do_eval\" : True\n",
        "}\n",
        "\n",
        "\n",
        "with open('args.json', 'w') as f:\n",
        "  json.dump(args_dict, f)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1MAYpR5IQjRH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 977
        },
        "outputId": "061c9283-a92b-4865-998e-b494a26f4ae6"
      },
      "source": [
        "main()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "06/15/2020 14:16:39 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:40 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:41 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:42 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:43 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:44 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:45 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:46 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:47 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:47 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:48 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:49 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:50 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:51 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:52 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:53 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:54 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:55 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:55 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:56 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:57 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:58 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:16:59 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:00 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:01 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:02 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:03 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:04 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:04 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:05 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:06 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:07 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:08 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:09 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:10 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:11 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:12 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:12 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:13 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:14 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:15 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:16 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:17 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 14:17:18 - INFO - transformers.trainer -   \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "06/15/2020 14:17:18 - INFO - transformers.trainer -   Saving model checkpoint to ./models\n",
            "06/15/2020 14:17:18 - INFO - transformers.configuration_utils -   Configuration saved in ./models/config.json\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "06/15/2020 14:17:20 - INFO - transformers.modeling_utils -   Model weights saved in ./models/pytorch_model.bin\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_8QXveQRXrpY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 801
        },
        "outputId": "70e5f644-8376-4225-dca7-9ee054e4dc56"
      },
      "source": [
        "tokenizer = LongformerTokenizerFast.from_pretrained(\n",
        "    \"allenai/longformer-base-4096\"\n",
        ")\n",
        "model = LongformerForSequenceClassification.from_pretrained(\n",
        "    \"./models\"\n",
        ")"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "06/15/2020 14:32:48 - INFO - transformers.tokenization_utils -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json from cache at /root/.cache/torch/transformers/1ae1f5b6e2b22b25ccc04c000bb79ca847aa226d0761536b011cf7e5868f0655.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
            "06/15/2020 14:32:48 - INFO - transformers.tokenization_utils -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txt from cache at /root/.cache/torch/transformers/f8f83199a6270d582d6245dc100e99c4155de81c9745c6248077018fe01abcfb.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
            "06/15/2020 14:32:49 - INFO - transformers.configuration_utils -   loading configuration file ./models/config.json\n",
            "06/15/2020 14:32:49 - INFO - transformers.configuration_utils -   Model config LongformerConfig {\n",
            "  \"architectures\": [\n",
            "    \"LongformerForSequenceClassification\"\n",
            "  ],\n",
            "  \"attention_mode\": \"longformer\",\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"attention_window\": [\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512,\n",
            "    512\n",
            "  ],\n",
            "  \"bos_token_id\": 0,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"ignore_attention_mask\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 4098,\n",
            "  \"model_type\": \"longformer\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"sep_token_id\": 2,\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "06/15/2020 14:32:49 - INFO - transformers.modeling_utils -   loading weights file ./models/pytorch_model.bin\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wA1hKEEJZax1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 817
        },
        "outputId": "9ef5df69-4f45-45e9-fbeb-1191cf95329f"
      },
      "source": [
        "model.cuda()\n",
        "model.eval()\n",
        "\n",
        "def simple_accuracy(preds, labels):\n",
        "  return (preds == labels).mean() \n",
        "\n",
        "label_ids: torch.Tensor = None\n",
        "preds: torch.Tensor = None\n",
        "\n",
        "from tqdm.auto import tqdm\n",
        "\n",
        "with torch.no_grad():\n",
        "  dataloader = torch.utils.data.DataLoader(valid_dataset, batch_size=16, collate_fn=DummyDataCollator().collate_batch)\n",
        "\n",
        "\n",
        "  for batch in tqdm(dataloader):\n",
        "    predictions = model(input_ids=batch['input_ids'].cuda(),\n",
        "                                  attention_mask=batch['attention_mask'].cuda())\n",
        "    \n",
        "    predictions = predictions[0]\n",
        "\n",
        "    if preds is None:\n",
        "      preds = predictions.detach()\n",
        "    else:\n",
        "      preds = torch.cat((preds, predictions.detach()), dim=0)\n",
        "\n",
        "\n",
        "    if label_ids is None:\n",
        "      label_ids = batch[\"labels\"].detach()\n",
        "    else:\n",
        "      label_ids = torch.cat((label_ids, batch[\"labels\"].detach()), dim=0)\n",
        "    \n",
        "\n",
        "\n",
        "preds = np.argmax(preds.cpu().numpy(), axis=1)\n",
        "label_ids = label_ids.cpu().numpy()\n",
        "\n",
        "print(simple_accuracy(preds, label_ids))"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "06/15/2020 15:36:08 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:08 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:09 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:09 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:10 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:10 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:11 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:11 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:11 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:12 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:12 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:13 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:13 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:14 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:14 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:15 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:15 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:16 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:16 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:16 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:17 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:17 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:18 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:18 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:19 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:19 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:20 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:20 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:21 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:21 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:21 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:22 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:22 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:23 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:23 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:24 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:24 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:25 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:25 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:26 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:26 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:26 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n",
            "06/15/2020 15:36:27 - INFO - transformers.modeling_longformer -   Initializing global attention on CLS token...\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "0.94996\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vKRygOiRxz7j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "VERSION = \"nightly\"  #@param [\"1.5\" , \"20200325\", \"nightly\"]\n",
        "!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n",
        "!python pytorch-xla-env-setup.py --version $VERSION"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Cx0EJErx5E0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 925
        },
        "outputId": "5f8cb5f8-8ed0-4a2e-e097-9cb33b1679a7"
      },
      "source": [
        "!pip install tf-nightly"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tf-nightly\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/45/9e/fd7c70c04f8a5794f2fe16190b8989baaa107d6b3c4265b92864d9db36b4/tf_nightly-2.3.0.dev20200615-cp36-cp36m-manylinux2010_x86_64.whl (350.2MB)\n",
            "\u001b[K     |████████████████████████████████| 350.2MB 47kB/s \n",
            "\u001b[?25hRequirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (1.29.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (1.12.0)\n",
            "Collecting tf-estimator-nightly\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fe/e1/337e99c3a46c3066c6678b0193c634d30df8adaade24a009f00b1a8eca41/tf_estimator_nightly-2.3.0.dev2020061501-py2.py3-none-any.whl (459kB)\n",
            "\u001b[K     |████████████████████████████████| 460kB 40.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (0.34.2)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (3.10.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (3.2.1)\n",
            "Collecting tb-nightly<2.4.0a0,>=2.3.0a0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/cf/cc/71f889286bf55e9cc5dceb6323adba6028d7f8890068edfac76149a5487b/tb_nightly-2.3.0a20200614-py3-none-any.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 43.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (1.1.2)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (0.3.3)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (1.6.3)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (0.9.0)\n",
            "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (1.18.5)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (1.1.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (0.2.0)\n",
            "Requirement already satisfied: scipy==1.4.1 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (1.4.1)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (2.10.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tf-nightly) (1.12.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.9.2->tf-nightly) (47.1.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (2.23.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (0.4.1)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (1.7.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (3.2.2)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (1.6.0.post3)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (1.0.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (2.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (2020.4.5.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (1.24.3)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (1.3.0)\n",
            "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (4.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (0.2.8)\n",
            "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (3.1.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (1.6.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (3.1.0)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (0.4.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly) (3.1.0)\n",
            "Installing collected packages: tf-estimator-nightly, tb-nightly, tf-nightly\n",
            "Successfully installed tb-nightly-2.3.0a20200614 tf-estimator-nightly-2.3.0.dev2020061501 tf-nightly-2.3.0.dev20200615\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "tensorboard",
                  "tensorflow"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IKO8DUalx6x4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        },
        "outputId": "55fc7668-03c3-455c-c826-32431b28c884"
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "import os\n",
        "import tensorflow_datasets as tfds\n",
        "\n",
        "resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])\n",
        "tf.config.experimental_connect_to_cluster(resolver)\n",
        "# This is the TPU initialization code that has to be at the beginning.\n",
        "tf.tpu.experimental.initialize_tpu_system(resolver)\n",
        "print(\"All devices: \", tf.config.list_logical_devices('TPU'))"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.82.205.42:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.82.205.42:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "All devices:  [LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:7', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:5', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:6', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:4', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:0', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:1', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:2', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:3', device_type='TPU')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6GjHamf4QrOf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "97139df82c7d45a29baf6f0e6f5f789c",
            "05a60c770f84443d8dc98f1f18d77408",
            "10aae66470f74182a0c62b9f65f34947",
            "21751f26145d4c0aa87181934585ce78",
            "5ce1ca7061074db2aac4b53628e022d1",
            "ac2ca3d91c9442a8912c8363626783fb",
            "8ea846c7b9f44087af1f7bb6022a0f84",
            "9af23bd3578b4089bd639fb1d5b36b1d",
            "cf2f4112938e4307be65719cb1f91416",
            "d1494b6ba2e54e93a29e0aaaaa37f14d",
            "5d3b1dfaeba24cf480f38fb5a48a9106",
            "dcdd52ac8bc141d9ac945e5ed0c01fff",
            "a529f5d01039461a86fd13d50de4ca0e",
            "1d7c226773aa4d1598b594d3d9de008f",
            "e8b84925accd4500b0296321367393a3",
            "ea9d7244c4144ba986b453d87780e217"
          ]
        },
        "outputId": "c7a88cc9-c7e1-4c75-d795-1b08d31e120e"
      },
      "source": [
        "import torch_xla.distributed.xla_multiprocessing as xmp\n",
        "xmp.spawn(_mp_fn, args=(), nprocs=8, start_method='fork')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:__main__:Process rank: -1, device: xla:1, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:transformers.training_args:Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n",
            "WARNING:transformers.training_args:Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "97139df82c7d45a29baf6f0e6f5f789c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cf2f4112938e4307be65719cb1f91416",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=391.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Exception in device=TPU:0: Error while lowering: f32[96,2,768,64]{3,2,1,0} aten::as_strided, size=(96, 2, 768, 64), stride=(65536, 16384, 64, 1), storage_offset=0\n",
            "Error: torch_xla/csrc/ops/as_strided.cpp:25 : Check failed: storage_offset + slice_size <= input_element_count (9437184 vs. 6291456)\n",
            "*** Begin stack trace ***\n",
            "\ttensorflow::CurrentStackTrace[abi:cxx11]()\n",
            "\ttorch_xla::ir::ops::AsStrided::Lower(torch_xla::ir::LoweringContext*) const\n",
            "\ttorch_xla::ir::LoweringContext::LowerNode(torch_xla::ir::Node const*)\n",
            "\ttorch_xla::ir::LoweringContext::LoweringContext(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, torch_xla::Device, absl::lts_2020_02_25::Span<torch_xla::ir::Node const* const>, std::unordered_map<torch_xla::ir::Node const*, torch_xla::ir::Util::EmitStatus, std::hash<torch_xla::ir::Node const*>, std::equal_to<torch_xla::ir::Node const*>, std::allocator<std::pair<torch_xla::ir::Node const* const, torch_xla::ir::Util::EmitStatus> > >)\n",
            "\ttorch_xla::XLATensor::Compile(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> > const&, absl::lts_2020_02_25::Span<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const>, torch_xla::XLATensor::SyncTensorCollection const&, torch_xla::XLATensor::PostOrderData*)\n",
            "\ttorch_xla::XLATensor::SyncTensorsGraphInternal(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> >*, absl::lts_2020_02_25::Span<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const>, torch_xla::XLATensor::SyncTensorsConfig const&)\n",
            "\ttorch_xla::XLATensor::GetTensorsFused(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> >*)\n",
            "\ttorch_xla::XLATensor::GetTensors(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> >*)\n",
            "\ttorch_xla::bridge::XlaCreateTensorList(c10::ArrayRef<at::Tensor> const&)\n",
            "\ttorch_xla::AtenXlaTypeDefault::_local_scalar_dense(at::Tensor const&)\n",
            "\tc10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoRuntimeFunctor_<c10::Scalar (*)(at::Tensor const&), c10::Scalar, c10::guts::typelist::typelist<at::Tensor const&> >, c10::Scalar (at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&)\n",
            "\t\n",
            "\tat::_local_scalar_dense(at::Tensor const&)\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\tat::_local_scalar_dense(at::Tensor const&)\n",
            "\tat::native::item(at::Tensor const&)\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\tat::Tensor::item() const\n",
            "\tdouble at::Tensor::item<double>() const\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t_PyFunction_FastCallDict\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyObject_FastCallKeywords\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t_PyFunction_FastCallDict\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t_PyFunction_FastCallDict\n",
            "\t\n",
            "*** End stack trace ***\n",
            "\n",
            "Python Frames:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch_xla/distributed/xla_multiprocessing.py\", line 231, in _start_fn\n",
            "    fn(gindex, *args)\n",
            "  File \"<ipython-input-13-17bd032f2529>\", line 180, in _mp_fn\n",
            "    main()\n",
            "  File \"<ipython-input-13-17bd032f2529>\", line 151, in main\n",
            "    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/transformers/trainer.py\", line 477, in train\n",
            "    tr_loss += self._training_step(model, inputs, optimizer)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/transformers/trainer.py\", line 604, in _training_step\n",
            "    return loss.item()\n",
            "RuntimeError: Error while lowering: f32[96,2,768,64]{3,2,1,0} aten::as_strided, size=(96, 2, 768, 64), stride=(65536, 16384, 64, 1), storage_offset=0\n",
            "Error: torch_xla/csrc/ops/as_strided.cpp:25 : Check failed: storage_offset + slice_size <= input_element_count (9437184 vs. 6291456)\n",
            "*** Begin stack trace ***\n",
            "\ttensorflow::CurrentStackTrace[abi:cxx11]()\n",
            "\ttorch_xla::ir::ops::AsStrided::Lower(torch_xla::ir::LoweringContext*) const\n",
            "\ttorch_xla::ir::LoweringContext::LowerNode(torch_xla::ir::Node const*)\n",
            "\ttorch_xla::ir::LoweringContext::LoweringContext(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, torch_xla::Device, absl::lts_2020_02_25::Span<torch_xla::ir::Node const* const>, std::unordered_map<torch_xla::ir::Node const*, torch_xla::ir::Util::EmitStatus, std::hash<torch_xla::ir::Node const*>, std::equal_to<torch_xla::ir::Node const*>, std::allocator<std::pair<torch_xla::ir::Node const* const, torch_xla::ir::Util::EmitStatus> > >)\n",
            "\ttorch_xla::XLATensor::Compile(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> > const&, absl::lts_2020_02_25::Span<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const>, torch_xla::XLATensor::SyncTensorCollection const&, torch_xla::XLATensor::PostOrderData*)\n",
            "\ttorch_xla::XLATensor::SyncTensorsGraphInternal(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> >*, absl::lts_2020_02_25::Span<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const>, torch_xla::XLATensor::SyncTensorsConfig const&)\n",
            "\ttorch_xla::XLATensor::GetTensorsFused(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> >*)\n",
            "\ttorch_xla::XLATensor::GetTensors(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> >*)\n",
            "\ttorch_xla::bridge::XlaCreateTensorList(c10::ArrayRef<at::Tensor> const&)\n",
            "\ttorch_xla::AtenXlaTypeDefault::_local_scalar_dense(at::Tensor const&)\n",
            "\tc10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoRuntimeFunctor_<c10::Scalar (*)(at::Tensor const&), c10::Scalar, c10::guts::typelist::typelist<at::Tensor const&> >, c10::Scalar (at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&)\n",
            "\t\n",
            "\tat::_local_scalar_dense(at::Tensor const&)\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\tat::_local_scalar_dense(at::Tensor const&)\n",
            "\tat::native::item(at::Tensor const&)\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\tat::Tensor::item() const\n",
            "\tdouble at::Tensor::item<double>() const\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t_PyFunction_FastCallDict\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyObject_FastCallKeywords\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t_PyFunction_FastCallDict\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\tPyObject_Call\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t\n",
            "\t\n",
            "\t_PyEval_EvalFrameDefault\n",
            "\t_PyFunction_FastCallDict\n",
            "\t\n",
            "*** End stack trace ***\n",
            "\n",
            "Python Frames:\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "Exception",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-bb90c4cdbb53>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch_xla\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistributed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxla_multiprocessing\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mxmp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mxmp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspawn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_mp_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnprocs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_method\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'fork'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch_xla/distributed/xla_multiprocessing.py\u001b[0m in \u001b[0;36mspawn\u001b[0;34m(fn, args, nprocs, join, daemon, start_method)\u001b[0m\n\u001b[1;32m    294\u001b[0m         \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m         \u001b[0mdaemon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdaemon\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 296\u001b[0;31m         start_method=start_method)\n\u001b[0m\u001b[1;32m    297\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/multiprocessing/spawn.py\u001b[0m in \u001b[0;36mstart_processes\u001b[0;34m(fn, args, nprocs, join, daemon, start_method)\u001b[0m\n\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;31m# Loop on join until it returns True or raises an exception.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 158\u001b[0;31m     \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    159\u001b[0m         \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/multiprocessing/spawn.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    111\u001b[0m                 raise Exception(\n\u001b[1;32m    112\u001b[0m                     \u001b[0;34m\"process %d terminated with exit code %d\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m                     \u001b[0;34m(\u001b[0m\u001b[0merror_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexitcode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    114\u001b[0m                 )\n\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mException\u001b[0m: process 0 terminated with exit code 17"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCMyKfFbyH-z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}