NLP_QA.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"NLP_QA.ipynb","provenance":[],"authorship_tag":"ABX9TyNqwy+VrHYlfW/781tjVpNW"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"d95cc3377ce4441290ffad6fcd4b7237":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_27bce849a8d74dd7a8466f3e3e059e23","IPY_MODEL_3c133148a2e740419db9787b4ffffe5b","IPY_MODEL_4a63b02a64754342ba1bfa5c9232bc36"],"layout":"IPY_MODEL_3e16e14ef464433585ffb4aed39c392f"}},"27bce849a8d74dd7a8466f3e3e059e23":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a432c1cd46904b93b12eb1e3fc6f8757","placeholder":"​","style":"IPY_MODEL_0cee5e4e49184e0c9c5c095b694973c4","value":"Downloading: 100%"}},"3c133148a2e740419db9787b4ffffe5b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8e7b8ae7a1af4c92969588270ce0b581","max":28,"min":0,"orientation":"horizontal","style":"IPY_MODEL_75789c02925040efa344967be025b913","value":28}},"4a63b02a64754342ba1bfa5c9232bc36":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_42e2288fffd84e54b60d8a5b265aa1f7","placeholder":"​","style":"IPY_MODEL_dd299fbc4aa64a9a8bc4f298e31d64a3","value":" 28.0/28.0 [00:00&lt;00:00, 611B/s]"}},"3e16e14ef464433585ffb4aed39c392f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a432c1cd46904b93b12eb1e3fc6f8757":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0cee5e4e49184e0c9c5c095b694973c4":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8e7b8ae7a1af4c92969588270ce0b581":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"75789c02925040efa344967be025b913":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"42e2288fffd84e54b60d8a5b265aa1f7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dd299fbc4aa64a9a8bc4f298e31d64a3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0e8e7e67ba0f434891922c9ad6912fef":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ea7fb5cdec6347d1b14a1a82f29d3ed0","IPY_MODEL_92863369a33e42189a53bc873acefdbb","IPY_MODEL_89abdd37db8e4d1384287e28697eac44"],"layout":"IPY_MODEL_f95e41d2faca490f84dcab0a103e8be5"}},"ea7fb5cdec6347d1b14a1a82f29d3ed0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_04b9432fee6e4d44a3c1b10c1c1a1f07","placeholder":"​","style":"IPY_MODEL_efca5e0cd6fa42ddaaaf2584dd365803","value":"Downloading: 100%"}},"92863369a33e42189a53bc873acefdbb":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_eb4936f26986426884f593c5daf020a1","max":231508,"min":0,"orientation":"horizontal","style":"IPY_MODEL_99f43ba846d140f4922ca67cb6ee462e","value":231508}},"89abdd37db8e4d1384287e28697eac44":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f594bdf05c3b4c9c8910f250b2d52e9f","placeholder":"​","style":"IPY_MODEL_03f7b44bc823485cbfdb3a822c0a8bd7","value":" 226k/226k [00:00&lt;00:00, 965kB/s]"}},"f95e41d2faca490f84dcab0a103e8be5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"04b9432fee6e4d44a3c1b10c1c1a1f07":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"efca5e0cd6fa42ddaaaf2584dd365803":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"eb4936f26986426884f593c5daf020a1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"99f43ba846d140f4922ca67cb6ee462e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f594bdf05c3b4c9c8910f250b2d52e9f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"03f7b44bc823485cbfdb3a822c0a8bd7":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"62831ba3b5284e8d9667eef8c6d81532":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_a759bc3889d349eb9cb7aafc851ad1f3","IPY_MODEL_ec15e667d7c44d3e8c5445802bede724","IPY_MODEL_98da5b6d7a324645ae95cf2ba0fcddb4"],"layout":"IPY_MODEL_7dc9de1eea444dbe9f37a4a91b8c1e6e"}},"a759bc3889d349eb9cb7aafc851ad1f3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_528ee1a93352479ea6962f20c1b8b947","placeholder":"​","style":"IPY_MODEL_58f7678be155442d8283c0a5014cb4dc","value":"Downloading: 100%"}},"ec15e667d7c44d3e8c5445802bede724":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_67ad439768ac4f1d84032a5a89e71529","max":466062,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0dc8506d0dec45ebb97a880599bbcfb6","value":466062}},"98da5b6d7a324645ae95cf2ba0fcddb4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ff77732b45844a5dba523608ddf2fddf","placeholder":"​","style":"IPY_MODEL_79e1304bfa4a4d83929a8e14562553c7","value":" 455k/455k [00:00&lt;00:00, 1.02MB/s]"}},"7dc9de1eea444dbe9f37a4a91b8c1e6e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"528ee1a93352479ea6962f20c1b8b947":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"58f7678be155442d8283c0a5014cb4dc":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"67ad439768ac4f1d84032a5a89e71529":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0dc8506d0dec45ebb97a880599bbcfb6":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ff77732b45844a5dba523608ddf2fddf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"79e1304bfa4a4d83929a8e14562553c7":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4970c8fc66604dbd9b605003b150d64b":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_215286d8694e41d7b4d140bcf08137d1","IPY_MODEL_c939856ec0064b3f85def18acf3f6270","IPY_MODEL_fde4de4d8cdd455facfbadd426d361ab"],"layout":"IPY_MODEL_0a346419fa5b46b88522090f30f19591"}},"215286d8694e41d7b4d140bcf08137d1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6df731259cdf4cc2b1a9ade1ae74fb6e","placeholder":"​","style":"IPY_MODEL_9db56437e56444109f9ab0d473541c35","value":"Downloading: 100%"}},"c939856ec0064b3f85def18acf3f6270":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_1f0d14823a0b471fb5f325444fcd2922","max":483,"min":0,"orientation":"horizontal","style":"IPY_MODEL_9b8eadc0011d4a21ba1ec9fa1c323d58","value":483}},"fde4de4d8cdd455facfbadd426d361ab":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8d6450fdbf974d07a0d1d99373200d72","placeholder":"​","style":"IPY_MODEL_d679a793bb5b4ac1b49f155184b69529","value":" 483/483 [00:00&lt;00:00, 12.7kB/s]"}},"0a346419fa5b46b88522090f30f19591":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6df731259cdf4cc2b1a9ade1ae74fb6e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9db56437e56444109f9ab0d473541c35":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1f0d14823a0b471fb5f325444fcd2922":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9b8eadc0011d4a21ba1ec9fa1c323d58":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8d6450fdbf974d07a0d1d99373200d72":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d679a793bb5b4ac1b49f155184b69529":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"83bab11aa8de4c499960a7e78cb9f45a":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_b9bf49ebf3e74f5abd2eecbe4280bab1","IPY_MODEL_4d2d4467ca0c42b9b272dc7994cd9b3e","IPY_MODEL_97a451cb8b624fd98903faa71eb1f411"],"layout":"IPY_MODEL_6a44a13ca92447c9a8268fc346516531"}},"b9bf49ebf3e74f5abd2eecbe4280bab1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d09fb64d63f9475c90f01e292ee55e94","placeholder":"​","style":"IPY_MODEL_84745853aa56434cb60532e191bbc956","value":"Downloading: 100%"}},"4d2d4467ca0c42b9b272dc7994cd9b3e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_fbd350614eb947ebb39d206c96c481f6","max":267967963,"min":0,"orientation":"horizontal","style":"IPY_MODEL_428fbe7dfe5947ca8724ad9365881b61","value":267967963}},"97a451cb8b624fd98903faa71eb1f411":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d67d54e147a3403daa6311761a50ef1f","placeholder":"​","style":"IPY_MODEL_87d7e96118d54b84b7a18705a96beb12","value":" 256M/256M [00:07&lt;00:00, 35.8MB/s]"}},"6a44a13ca92447c9a8268fc346516531":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d09fb64d63f9475c90f01e292ee55e94":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"84745853aa56434cb60532e191bbc956":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"fbd350614eb947ebb39d206c96c481f6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"428fbe7dfe5947ca8724ad9365881b61":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"d67d54e147a3403daa6311761a50ef1f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"87d7e96118d54b84b7a18705a96beb12":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"markdown","source":["How to build Custom Q&A transformer models in Python from https://www.youtube.com/watch?v=ZIRmXkHp0-c&ab_channel=JamesBriggs"],"metadata":{"id":"P1yAfnE8MXNz"}},{"cell_type":"code","source":["!pip install transformers"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"oom4FrbyheNI","executionInfo":{"status":"ok","timestamp":1654787057674,"user_tz":240,"elapsed":10449,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"6780f483-89eb-4d49-f24b-d2b3f61c809f"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting transformers\n","  Downloading transformers-4.19.2-py3-none-any.whl (4.2 MB)\n","\u001b[K     |████████████████████████████████| 4.2 MB 8.4 MB/s \n","\u001b[?25hCollecting tokenizers!=0.11.3,<0.13,>=0.11.1\n","  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n","\u001b[K     |████████████████████████████████| 6.6 MB 35.6 MB/s \n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n","Collecting pyyaml>=5.1\n","  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n","\u001b[K     |████████████████████████████████| 596 kB 44.7 MB/s \n","\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.4)\n","Collecting huggingface-hub<1.0,>=0.1.0\n","  Downloading huggingface_hub-0.7.0-py3-none-any.whl (86 kB)\n","\u001b[K     |████████████████████████████████| 86 kB 6.0 MB/s \n","\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.5.18.1)\n","Installing collected packages: pyyaml, tokenizers, huggingface-hub, transformers\n","  Attempting uninstall: pyyaml\n","    Found existing installation: PyYAML 3.13\n","    Uninstalling PyYAML-3.13:\n","      Successfully uninstalled PyYAML-3.13\n","Successfully installed huggingface-hub-0.7.0 pyyaml-6.0 tokenizers-0.12.1 transformers-4.19.2\n"]}]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/gdrive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"qOvbNWlMMgQn","executionInfo":{"status":"ok","timestamp":1654787076770,"user_tz":240,"elapsed":17559,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"fd39ca5f-03c3-4c22-de81-29ef38bac0d2"},"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/gdrive\n"]}]},{"cell_type":"code","source":["import os"],"metadata":{"id":"KBApnQE3NRj9","executionInfo":{"status":"ok","timestamp":1654787078919,"user_tz":240,"elapsed":142,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["!mkdir squad"],"metadata":{"id":"CGDjuBVgOv3y","executionInfo":{"status":"ok","timestamp":1654787080791,"user_tz":240,"elapsed":181,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["%cd squad"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Pl9HnqcaO3au","executionInfo":{"status":"ok","timestamp":1654787086478,"user_tz":240,"elapsed":3,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"21872b9b-ef79-455c-9de9-595255a422da"},"execution_count":9,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/squad\n"]}]},{"cell_type":"code","source":["url = 'https://rajpurkar.github.io/SQuAD-explorer/dataset/'"],"metadata":{"id":"RmQET1csO6Hb","executionInfo":{"status":"ok","timestamp":1654787083730,"user_tz":240,"elapsed":124,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["import requests"],"metadata":{"id":"39J0C8z6RRIB","executionInfo":{"status":"ok","timestamp":1654787088532,"user_tz":240,"elapsed":161,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":10,"outputs":[]},{"cell_type":"code","source":["res = requests.get(f'{url}train-v2.0.json')"],"metadata":{"id":"egrlkoLbRSNE","executionInfo":{"status":"ok","timestamp":1654787091172,"user_tz":240,"elapsed":2028,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","source":["for file in ['train-v2.0.json', 'dev-v2.0.json']:\n","    res = requests.get(f'{url}{file}')\n","    # write to file\n","    with open(f'{file}', 'wb') as f:\n","        for chunk in res.iter_content(chunk_size=4):\n","            f.write(chunk)"],"metadata":{"id":"b6I_JGu7Syjn","executionInfo":{"status":"ok","timestamp":1654787098400,"user_tz":240,"elapsed":7232,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":12,"outputs":[]},{"cell_type":"markdown","source":["Data prep"],"metadata":{"id":"0dWTa82YTk7V"}},{"cell_type":"code","source":["import json"],"metadata":{"id":"b5M7EGwATnSf","executionInfo":{"status":"ok","timestamp":1654787098628,"user_tz":240,"elapsed":232,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":13,"outputs":[]},{"cell_type":"code","source":["def read_squad(path):\n","  with open(path, 'rb') as f:\n","    squad_dict = json.load(f)\n","  contexts = []\n","  questions = []\n","  answers = []\n","\n","  for group in squad_dict['data']:\n","    for passage in group['paragraphs']:\n","      context= passage['context']\n","      for qa in passage['qas']:\n","        question = qa['question']\n","        if 'plausible_answers' in qa.keys():\n","          access = 'plausible_answers'\n","        else:\n","          access = 'answers'\n","        for answer in qa[access]:\n","          contexts.append(context)\n","          questions.append(question)\n","          answers.append(answer)\n","  return contexts,questions,answers\n","    \n","    "],"metadata":{"id":"t_JmXQpjY4hQ","executionInfo":{"status":"ok","timestamp":1654787113422,"user_tz":240,"elapsed":138,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":15,"outputs":[]},{"cell_type":"code","source":["train_contexts,train_questions,train_answers= read_squad('train-v2.0.json')\n","val_contexts,val_questions,val_answers= read_squad('dev-v2.0.json')"],"metadata":{"id":"nm8k6AemY5BW","executionInfo":{"status":"ok","timestamp":1654787116620,"user_tz":240,"elapsed":1214,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":16,"outputs":[]},{"cell_type":"code","source":["train_answers[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"aGpHtyEEbuW4","executionInfo":{"status":"ok","timestamp":1654787117831,"user_tz":240,"elapsed":144,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"0b459b44-b587-4986-cee3-3e02bcb3644f"},"execution_count":17,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'answer_start': 269, 'text': 'in the late 1990s'}"]},"metadata":{},"execution_count":17}]},{"cell_type":"code","source":["def add_end_idx(answers,contexts):\n","  for answer,context in zip(answers,contexts):\n","    gold_text=answer['text']\n","    start_idx=answer['answer_start']\n","    end_idx=start_idx+ len(gold_text) \n","\n","    if context[start_idx:end_idx] == gold_text:\n","      answer['answer_end'] = end_idx\n","    else:\n","      for n in [1,2]:\n","        if context[start_idx-n:end_idx-n] == gold_text:\n","          answer['answer_start'] = start_idx-n\n","          answer['answer_end'] = end_idx-n\n","\n","add_end_idx(train_answers,train_contexts)\n","add_end_idx(val_answers,val_contexts) \n"," "],"metadata":{"id":"Be5-r3nwb9jZ","executionInfo":{"status":"ok","timestamp":1654787119426,"user_tz":240,"elapsed":125,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["train_answers[:5]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"oLlxCoWPg-dQ","executionInfo":{"status":"ok","timestamp":1654787121353,"user_tz":240,"elapsed":145,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"6afa12c3-0077-4cf3-922f-ed9bd45b76ba"},"execution_count":19,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[{'answer_end': 286, 'answer_start': 269, 'text': 'in the late 1990s'},\n"," {'answer_end': 226, 'answer_start': 207, 'text': 'singing and dancing'},\n"," {'answer_end': 530, 'answer_start': 526, 'text': '2003'},\n"," {'answer_end': 180, 'answer_start': 166, 'text': 'Houston, Texas'},\n"," {'answer_end': 286, 'answer_start': 276, 'text': 'late 1990s'}]"]},"metadata":{},"execution_count":19}]},{"cell_type":"markdown","source":["Tokenize/Encode"],"metadata":{"id":"pmx0FnwVhKYu"}},{"cell_type":"code","source":["from transformers import DistilBertTokenizerFast"],"metadata":{"id":"A7nO9f1LhL3Z","executionInfo":{"status":"ok","timestamp":1654787126970,"user_tz":240,"elapsed":3726,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":20,"outputs":[]},{"cell_type":"code","source":["tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased') "],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":145,"referenced_widgets":["d95cc3377ce4441290ffad6fcd4b7237","27bce849a8d74dd7a8466f3e3e059e23","3c133148a2e740419db9787b4ffffe5b","4a63b02a64754342ba1bfa5c9232bc36","3e16e14ef464433585ffb4aed39c392f","a432c1cd46904b93b12eb1e3fc6f8757","0cee5e4e49184e0c9c5c095b694973c4","8e7b8ae7a1af4c92969588270ce0b581","75789c02925040efa344967be025b913","42e2288fffd84e54b60d8a5b265aa1f7","dd299fbc4aa64a9a8bc4f298e31d64a3","0e8e7e67ba0f434891922c9ad6912fef","ea7fb5cdec6347d1b14a1a82f29d3ed0","92863369a33e42189a53bc873acefdbb","89abdd37db8e4d1384287e28697eac44","f95e41d2faca490f84dcab0a103e8be5","04b9432fee6e4d44a3c1b10c1c1a1f07","efca5e0cd6fa42ddaaaf2584dd365803","eb4936f26986426884f593c5daf020a1","99f43ba846d140f4922ca67cb6ee462e","f594bdf05c3b4c9c8910f250b2d52e9f","03f7b44bc823485cbfdb3a822c0a8bd7","62831ba3b5284e8d9667eef8c6d81532","a759bc3889d349eb9cb7aafc851ad1f3","ec15e667d7c44d3e8c5445802bede724","98da5b6d7a324645ae95cf2ba0fcddb4","7dc9de1eea444dbe9f37a4a91b8c1e6e","528ee1a93352479ea6962f20c1b8b947","58f7678be155442d8283c0a5014cb4dc","67ad439768ac4f1d84032a5a89e71529","0dc8506d0dec45ebb97a880599bbcfb6","ff77732b45844a5dba523608ddf2fddf","79e1304bfa4a4d83929a8e14562553c7","4970c8fc66604dbd9b605003b150d64b","215286d8694e41d7b4d140bcf08137d1","c939856ec0064b3f85def18acf3f6270","fde4de4d8cdd455facfbadd426d361ab","0a346419fa5b46b88522090f30f19591","6df731259cdf4cc2b1a9ade1ae74fb6e","9db56437e56444109f9ab0d473541c35","1f0d14823a0b471fb5f325444fcd2922","9b8eadc0011d4a21ba1ec9fa1c323d58","8d6450fdbf974d07a0d1d99373200d72","d679a793bb5b4ac1b49f155184b69529"]},"id":"JSUkzhGGhTuD","executionInfo":{"status":"ok","timestamp":1654787131541,"user_tz":240,"elapsed":3852,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"ce5a48c6-a64f-4b7b-d55e-ba4f5a582668"},"execution_count":21,"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d95cc3377ce4441290ffad6fcd4b7237"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0e8e7e67ba0f434891922c9ad6912fef"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"62831ba3b5284e8d9667eef8c6d81532"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/483 [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4970c8fc66604dbd9b605003b150d64b"}},"metadata":{}}]},{"cell_type":"code","source":["train_encodings= tokenizer(train_contexts,train_questions, truncation=True, padding=True)\n","val_encodings= tokenizer(val_contexts,val_questions, truncation=True, padding=True)"],"metadata":{"id":"HSTzVgOOh3LD","executionInfo":{"status":"ok","timestamp":1654787204222,"user_tz":240,"elapsed":71231,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":22,"outputs":[]},{"cell_type":"code","source":["train_encodings.keys()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4oiEpdBziTPH","executionInfo":{"status":"ok","timestamp":1654787204223,"user_tz":240,"elapsed":23,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"b8fc40ea-f90d-4f81-fe6c-08b599614324"},"execution_count":23,"outputs":[{"output_type":"execute_result","data":{"text/plain":["dict_keys(['input_ids', 'attention_mask'])"]},"metadata":{},"execution_count":23}]},{"cell_type":"code","source":["tokenizer.decode(train_encodings['input_ids'][0])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":157},"id":"x4siZnr_vF9q","executionInfo":{"status":"ok","timestamp":1654787204224,"user_tz":240,"elapsed":9,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"7dc6c647-90dd-4912-f463-bbed5c3b2cfd"},"execution_count":24,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'[CLS] beyonce giselle knowles - carter ( / biːˈjɒnseɪ / bee - yon - say ) ( born september 4, 1981 ) is an american singer, songwriter, record producer and actress. born and raised in houston, texas, she performed in various singing and dancing competitions as a child, and rose to fame in the late 1990s as lead singer of r & b girl - group destiny\\'s child. managed by her father, mathew knowles, the group became one of the world\\'s best - selling girl groups of all time. their hiatus saw the release of beyonce\\'s debut album, dangerously in love ( 2003 ), which established her as a solo artist worldwide, earned five grammy awards and featured the billboard hot 100 number - one singles \" crazy in love \" and \" baby boy \". [SEP] when did beyonce start becoming popular? [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":24}]},{"cell_type":"code","source":["def add_token_positions(encodings,answers):\n","  start_positions=[]\n","  end_positions=[]\n","  for i in range(len(answers)):\n","    start_positions.append(encodings.char_to_token(i,answers[i]['answer_start']))\n","    end_positions.append(encodings.char_to_token(i,answers[i]['answer_end']))\n","    if start_positions[-1] is None:\n","      start_positions[-1] = tokenizer.model_max_length\n","    go_back=1\n","    while end_positions[-1] is None:\n","      end_positions[-1] = encodings.char_to_token(i,answers[i]['answer_end']-go_back)\n","      go_back+=1\n","  encodings.update({\n","      'start_positions': start_positions,\n","      'end_positions': end_positions\n","  })\n","\n","add_token_positions(train_encodings,train_answers)\n","add_token_positions(val_encodings,val_answers)\n"],"metadata":{"id":"PPPREFOpvHHN","executionInfo":{"status":"ok","timestamp":1654788517253,"user_tz":240,"elapsed":933,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":37,"outputs":[]},{"cell_type":"code","source":["import torch\n","\n","class squadDataset(torch.utils.data.Dataset):\n","  def __init__(self,encodings):\n","    self.encodings = encodings\n","  def __getitem__(self,idx):\n","    return {key: torch.tensor(val[idx]) for key,val in self.encodings.items()}\n","  def __len__(self):\n","    return len(self.encodings.input_ids)"],"metadata":{"id":"hoeiO4Xa0JxO","executionInfo":{"status":"ok","timestamp":1654787211609,"user_tz":240,"elapsed":255,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":29,"outputs":[]},{"cell_type":"code","source":["train_datasets = squadDataset(train_encodings)\n","val_datasets = squadDataset(val_encodings)\n"],"metadata":{"id":"TaVuW1wt0yV8","executionInfo":{"status":"ok","timestamp":1654787212754,"user_tz":240,"elapsed":126,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":30,"outputs":[]},{"cell_type":"markdown","source":["Fine-Tuning"],"metadata":{"id":"XVmyNQjJ48-g"}},{"cell_type":"code","source":["from transformers import DistilBertForQuestionAnswering\n","model = DistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":156,"referenced_widgets":["83bab11aa8de4c499960a7e78cb9f45a","b9bf49ebf3e74f5abd2eecbe4280bab1","4d2d4467ca0c42b9b272dc7994cd9b3e","97a451cb8b624fd98903faa71eb1f411","6a44a13ca92447c9a8268fc346516531","d09fb64d63f9475c90f01e292ee55e94","84745853aa56434cb60532e191bbc956","fbd350614eb947ebb39d206c96c481f6","428fbe7dfe5947ca8724ad9365881b61","d67d54e147a3403daa6311761a50ef1f","87d7e96118d54b84b7a18705a96beb12"]},"id":"AXSQP35G6smv","executionInfo":{"status":"ok","timestamp":1654787223599,"user_tz":240,"elapsed":9398,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"a186ec38-70ed-4699-83c3-4bfcf7360a2c"},"execution_count":31,"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/256M [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"83bab11aa8de4c499960a7e78cb9f45a"}},"metadata":{}},{"output_type":"stream","name":"stderr","text":["Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForQuestionAnswering: ['vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_layer_norm.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n","- This IS expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","Some weights of DistilBertForQuestionAnswering were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]}]},{"cell_type":"code","source":["from torch.utils.data import DataLoader\n","from transformers import AdamW\n","from tqdm import tqdm"],"metadata":{"id":"thtoedcy4_cI","executionInfo":{"status":"ok","timestamp":1654787225651,"user_tz":240,"elapsed":122,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":32,"outputs":[]},{"cell_type":"code","source":["device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n","model.to(device)\n","model.train()\n","optim = AdamW(model.parameters(), lr=5e-5)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2Fo5n4WE5Jbi","executionInfo":{"status":"ok","timestamp":1654787245793,"user_tz":240,"elapsed":17755,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"1146cadd-ac83-464f-8ef2-5e9e3d2c5ea3"},"execution_count":33,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n","  FutureWarning,\n"]}]},{"cell_type":"code","source":["train_loader = DataLoader(train_datasets, batch_size=16,shuffle=True)"],"metadata":{"id":"E3hsISeg6oMI","executionInfo":{"status":"ok","timestamp":1654787251586,"user_tz":240,"elapsed":129,"user":{"displayName":"Juan D","userId":"01890547855168746202"}}},"execution_count":34,"outputs":[]},{"cell_type":"code","source":["for epoch in range(3):\n","  loop = tqdm(train_loader)\n","  for batch in loop:\n","    optim.zero_grad()\n","\n","    input_ids = batch['input_ids'].to(device)\n","    attention_mask = batch['attention_mask'].to(device)\n","    start_positions = batch['start_positions'].to(device)\n","    end_positions = batch['end_positions'].to(device)\n","\n","    outputs = model(input_ids, attention_mask = attention_mask,\n","                    start_positions = start_positions,\n","                    end_positions = end_positions)\n","    loss = outputs[0]\n","    loss.backward()\n","    optim.step()\n","\n","    loop.set_description(f'Epoch{epoch}')\n","    loop.set_postfix(loss=loss.item())"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kwJIQ7sm7otQ","executionInfo":{"status":"ok","timestamp":1654799095645,"user_tz":240,"elapsed":10574186,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"8ab30f0f-0c85-4d4d-a112-ebd68ca91cf0"},"execution_count":38,"outputs":[{"output_type":"stream","name":"stderr","text":["Epoch0: 100%|██████████| 8145/8145 [58:46<00:00,  2.31it/s, loss=0.926]\n","Epoch1: 100%|██████████| 8145/8145 [58:42<00:00,  2.31it/s, loss=0.76]\n","Epoch2: 100%|██████████| 8145/8145 [58:45<00:00,  2.31it/s, loss=0.615]\n"]}]},{"cell_type":"code","source":["model_path='model/distibert-custom'\n","model.save_pretrained(model_path)\n","tokenizer.save_pretrained(model_path)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rAxj5p6cFr_i","executionInfo":{"status":"ok","timestamp":1654799386932,"user_tz":240,"elapsed":1489,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"d5f4781f-19f5-4c46-a262-3f4e6a612e22"},"execution_count":39,"outputs":[{"output_type":"execute_result","data":{"text/plain":["('model/distibert-custom/tokenizer_config.json',\n"," 'model/distibert-custom/special_tokens_map.json',\n"," 'model/distibert-custom/vocab.txt',\n"," 'model/distibert-custom/added_tokens.json',\n"," 'model/distibert-custom/tokenizer.json')"]},"metadata":{},"execution_count":39}]},{"cell_type":"code","source":["model.eval()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6nk8wtcJGNoU","executionInfo":{"status":"ok","timestamp":1654799388654,"user_tz":240,"elapsed":151,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"9070befb-23d7-4b45-fb48-c76a08441632"},"execution_count":40,"outputs":[{"output_type":"execute_result","data":{"text/plain":["DistilBertForQuestionAnswering(\n","  (distilbert): DistilBertModel(\n","    (embeddings): Embeddings(\n","      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n","      (position_embeddings): Embedding(512, 768)\n","      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","      (dropout): Dropout(p=0.1, inplace=False)\n","    )\n","    (transformer): Transformer(\n","      (layer): ModuleList(\n","        (0): TransformerBlock(\n","          (attention): MultiHeadSelfAttention(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","          (ffn): FFN(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n","            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n","            (activation): GELUActivation()\n","          )\n","          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","        )\n","        (1): TransformerBlock(\n","          (attention): MultiHeadSelfAttention(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","          (ffn): FFN(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n","            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n","            (activation): GELUActivation()\n","          )\n","          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","        )\n","        (2): TransformerBlock(\n","          (attention): MultiHeadSelfAttention(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","          (ffn): FFN(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n","            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n","            (activation): GELUActivation()\n","          )\n","          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","        )\n","        (3): TransformerBlock(\n","          (attention): MultiHeadSelfAttention(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","          (ffn): FFN(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n","            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n","            (activation): GELUActivation()\n","          )\n","          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","        )\n","        (4): TransformerBlock(\n","          (attention): MultiHeadSelfAttention(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","          (ffn): FFN(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n","            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n","            (activation): GELUActivation()\n","          )\n","          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","        )\n","        (5): TransformerBlock(\n","          (attention): MultiHeadSelfAttention(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","          (ffn): FFN(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n","            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n","            (activation): GELUActivation()\n","          )\n","          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","        )\n","      )\n","    )\n","  )\n","  (qa_outputs): Linear(in_features=768, out_features=2, bias=True)\n","  (dropout): Dropout(p=0.1, inplace=False)\n",")"]},"metadata":{},"execution_count":40}]},{"cell_type":"code","source":["val_loader= DataLoader(val_datasets, batch_size=16)\n","acc=[]\n","\n","loop = tqdm(val_loader)\n","for batch in loop:\n","  with torch.no_grad():\n","    input_ids = batch['input_ids'].to(device)\n","    attention_mask = batch['attention_mask'].to(device)\n","    start_true = batch['start_positions'].to(device)\n","    end_true = batch['end_positions'].to(device)\n","\n","    outputs = model(input_ids, attention_mask = attention_mask)\n","\n","    start_pred=torch.argmax(outputs['start_logits'], dim=1)\n","    end_pred=torch.argmax(outputs['end_logits'], dim=1)\n","\n","    acc.append(((start_pred==start_true).sum()/len(start_pred)).item())\n","    acc.append(((end_pred==end_true).sum()/len(end_pred)).item())\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Of_sRX1QGOrp","executionInfo":{"status":"ok","timestamp":1654799624903,"user_tz":240,"elapsed":230320,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"6f3eb173-018d-4332-aac3-f93d0b3e542b"},"execution_count":41,"outputs":[{"output_type":"stream","name":"stderr","text":["100%|██████████| 1640/1640 [03:48<00:00,  7.17it/s]\n"]}]},{"cell_type":"code","source":["sum(acc)/len(acc)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0r5nMQx1Kcj-","executionInfo":{"status":"ok","timestamp":1654799761748,"user_tz":240,"elapsed":187,"user":{"displayName":"Juan D","userId":"01890547855168746202"}},"outputId":"5adb54bd-f07d-4f3e-e001-c3cb73c89522"},"execution_count":42,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.6413681402439024"]},"metadata":{},"execution_count":42}]}]}