train_forward_translation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "unformated_en_2_formated_fr_w2w.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "nb67W9VFEWrq"
      ],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3719dc5d1d1e47f2a84b75194970f11b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f99de86c50da449ebaa87ff8e172e74a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1463cbfaf0544e418e197e35c565f612",
              "IPY_MODEL_450677d6c74b45ee83902dbf96f1a257"
            ]
          }
        },
        "f99de86c50da449ebaa87ff8e172e74a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1463cbfaf0544e418e197e35c565f612": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4ef048fad07a4772940f209a95c04eec",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f4bd0d5f3394a479043a45cebc12a0a"
          }
        },
        "450677d6c74b45ee83902dbf96f1a257": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_117ea04694884baa92c55e70abc04310",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [10:09&lt;00:00, 61.00s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_14b4dd74a5744adda8dd07bc3a8cb2b8"
          }
        },
        "4ef048fad07a4772940f209a95c04eec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f4bd0d5f3394a479043a45cebc12a0a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "117ea04694884baa92c55e70abc04310": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "14b4dd74a5744adda8dd07bc3a8cb2b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a88e732e81b447eaa7db435501e468c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_acf00b053c1d4ea7acf3963728bf0471",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4d9a3801f9fd48859cbecb63c07f948d",
              "IPY_MODEL_ffef32d4364d4b9da9fc03f739f0e3c6"
            ]
          }
        },
        "acf00b053c1d4ea7acf3963728bf0471": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d9a3801f9fd48859cbecb63c07f948d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d62751787d8a4547859e76249cb1070a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de37452ce97645acb3ca9423781cdd0b"
          }
        },
        "ffef32d4364d4b9da9fc03f739f0e3c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_179b8510943f49de828a367820fdb184",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 7/? [01:23&lt;00:00, 11.87s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b4e1dcc68e2043c1a8f91bb69e7f3474"
          }
        },
        "d62751787d8a4547859e76249cb1070a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de37452ce97645acb3ca9423781cdd0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "179b8510943f49de828a367820fdb184": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b4e1dcc68e2043c1a8f91bb69e7f3474": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e6c7cbf209c24264a30465c2c272fd5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7adac9bb16bc4acf8796aefe21ebd508",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_026da294751543e5946394f90a107d12",
              "IPY_MODEL_4b30a7016fd24f17973fd5b2c4f80636"
            ]
          }
        },
        "7adac9bb16bc4acf8796aefe21ebd508": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "026da294751543e5946394f90a107d12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ee4eb2bae55f4c3fb6f49bc801697e42",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 400,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 400,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ead005a0cb024ce5bbde87e4339b5eb9"
          }
        },
        "4b30a7016fd24f17973fd5b2c4f80636": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3a910fe9cd374eb6aa39cd61d7309ca2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1852/? [54:04&lt;00:00,  1.75s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1bc9ca6a36cb4e28893932dc9a0cb737"
          }
        },
        "ee4eb2bae55f4c3fb6f49bc801697e42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ead005a0cb024ce5bbde87e4339b5eb9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3a910fe9cd374eb6aa39cd61d7309ca2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1bc9ca6a36cb4e28893932dc9a0cb737": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "s_qNSzzyaCbD"
      },
      "source": [
        "##### Copyright 2019 The TensorFlow Authors."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "jmjh290raIky",
        "colab": {}
      },
      "source": [
        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sxLx0cxT_fJ7",
        "colab_type": "text"
      },
      "source": [
        "### Colab stuff"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BWqhS7vxL8sr",
        "colab_type": "code",
        "outputId": "ec15fbae-6e54-43b5-991d-c7c4c67f1881",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oqoNO6SIMIir",
        "colab_type": "code",
        "outputId": "c0dbe75c-7e24-45e4-ec75-de8205ac2565",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        }
      },
      "source": [
        "!pip install gensim==3.8.1\n",
        "!pip install sacrebleu"
      ],
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: gensim==3.8.1 in /usr/local/lib/python3.6/dist-packages (3.8.1)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim==3.8.1) (1.18.2)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim==3.8.1) (1.4.1)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.6/dist-packages (from gensim==3.8.1) (1.11.1)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim==3.8.1) (1.12.0)\n",
            "Requirement already satisfied: boto in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.8.1->gensim==3.8.1) (2.49.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.8.1->gensim==3.8.1) (2.21.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.8.1->gensim==3.8.1) (1.12.39)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.8.1->gensim==3.8.1) (3.0.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.8.1->gensim==3.8.1) (2.8)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.8.1->gensim==3.8.1) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.8.1->gensim==3.8.1) (2020.4.5.1)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.8.1->gensim==3.8.1) (0.9.5)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.39 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.8.1->gensim==3.8.1) (1.15.39)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.8.1->gensim==3.8.1) (0.3.3)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.39->boto3->smart-open>=1.8.1->gensim==3.8.1) (2.8.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.39->boto3->smart-open>=1.8.1->gensim==3.8.1) (0.15.2)\n",
            "Requirement already satisfied: sacrebleu in /usr/local/lib/python3.6/dist-packages (1.4.7)\n",
            "Requirement already satisfied: mecab-python3 in /usr/local/lib/python3.6/dist-packages (from sacrebleu) (0.996.5)\n",
            "Requirement already satisfied: typing in /usr/local/lib/python3.6/dist-packages (from sacrebleu) (3.6.6)\n",
            "Requirement already satisfied: portalocker in /usr/local/lib/python3.6/dist-packages (from sacrebleu) (1.7.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sWdkK1xIL5IO",
        "colab_type": "code",
        "outputId": "7e8504d7-52c0-43f1-e2e6-fe626eaedddc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import os\n",
        "os.chdir(os.path.join('drive', 'My Drive', 'ift6759', 'ift6759-t6-p2'))\n",
        "os.getcwd()"
      ],
      "execution_count": 147,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/drive/My Drive/ift6759/ift6759-t6-p2'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 147
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rYDRwI_X_oSc",
        "colab_type": "text"
      },
      "source": [
        "### imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BKXrS5VKKORC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import os\n",
        "import json\n",
        "import time\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import sacrebleu\n",
        "\n",
        "from tqdm.notebook import tqdm\n",
        "from sklearn.model_selection import train_test_split\n",
        "from dataloader.dataloader import get_dataset_train\n",
        "from utils.data import preprocess_v2id, postprocessing\n",
        "from utils.data import save_json, load_json, swap_dict_key_value, checkout_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V3SG8WxB_q4E",
        "colab_type": "text"
      },
      "source": [
        "### Dataloader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uAt-tC85KwQ2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def encoder_preprocess(data):\n",
        "    encoder_v2id, encoder_dataset = preprocess_v2id(\n",
        "            data=data,\n",
        "            v2id=os.path.join(\"language_models\",\"unformated_en_w2w\", \"v2id.json\"),\n",
        "            tokenize_type=\"w\",\n",
        "            max_seq=None,\n",
        "            remove_punctuation=True,\n",
        "            lower=False,\n",
        "            threshold=0.85,\n",
        "            CAP=True,\n",
        "            NUM=True,\n",
        "            ALNUM=True,\n",
        "            UPPER=True,\n",
        "            fasttext_model=\"/content/drive/My Drive/ift6759/ift6759-t6-p2/embeddings/unformated_en_w2w/128/unaligned_unformated_en\",\n",
        "    )\n",
        "    encoder_id2v = swap_dict_key_value(encoder_v2id)\n",
        "    return encoder_dataset, encoder_v2id, encoder_id2v\n",
        "\n",
        "def decoder_preprocess(data):\n",
        "    decoder_v2id, decoder_dataset = preprocess_v2id(\n",
        "            data=data,\n",
        "            v2id=os.path.join(\"language_models\",\"formated_fr_w2w\", \"v2id.json\"),\n",
        "            tokenize_type=\"w\",\n",
        "            max_seq=None,\n",
        "            remove_punctuation=False,\n",
        "            lower=False,\n",
        "            threshold=0.85,\n",
        "            CAP=True,\n",
        "            NUM=True,\n",
        "            ALNUM=True,\n",
        "            UPPER=True,\n",
        "            fasttext_model=\"/content/drive/My Drive/ift6759/ift6759-t6-p2/embeddings/formated_fr_w2w/128/unaligned_formated_fr\",\n",
        "    )\n",
        "    decoder_id2v = swap_dict_key_value(decoder_v2id)\n",
        "    return decoder_dataset, decoder_v2id, decoder_id2v\n",
        "\n",
        "def postprocess(out, encoder_v2id, decoder_v2id, enc_data_words):\n",
        "    func = postprocessing(\n",
        "                dec_data=out,\n",
        "                dec_v2id=decoder_v2id,\n",
        "                Print=False,\n",
        "                tokenize_type=\"w\",\n",
        "                fasttext_model=\"/content/drive/My Drive/ift6759/ift6759-t6-p2/embeddings/unformated_en_w2w/128/unaligned_unformated_en\",\n",
        "                enc_data=enc_data_words,\n",
        "                remove_punctuation=True,\n",
        "                lower=False,\n",
        "                CAP=True,\n",
        "                NUM=True,\n",
        "                ALNUM=True,\n",
        "                UPPER=True,\n",
        "                enc_v2id=encoder_v2id\n",
        "            )\n",
        "    return func\n",
        "\n",
        "def get_gen(data, batch_size, valid_test=False):\n",
        "    def data_word_generator():\n",
        "        ch_data = checkout_data(data)\n",
        "        if valid_test:\n",
        "          ch_data = ch_data[int(-len(ch_data)*0.02):]\n",
        "        size = len(ch_data)\n",
        "        steps = size//batch_size + 1\n",
        "        init = 0\n",
        "        end = batch_size\n",
        "        for i in range(steps):\n",
        "            to_return = ch_data[init:end]\n",
        "            init = end\n",
        "            end += batch_size\n",
        "            yield to_return\n",
        "    return data_word_generator()\n",
        "\n",
        "def get_out(enc_data_words):\n",
        "    enc_data_int, _, _ = encoder_preprocess(data=enc_data_words)\n",
        "    out, _ = evaluate(enc_data_int)\n",
        "    out_words = postprocess(out, encoder_v2id, decoder_v2id, enc_data_words)\n",
        "    return out_words\n",
        "\n",
        "def write_file_append(data, output):\n",
        "    with open(output, 'a+') as stream:\n",
        "      for line in data:\n",
        "        stream.write(line)\n",
        "        stream.write(\"\\n\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "13Ttej-7Wua4",
        "colab_type": "text"
      },
      "source": [
        "**real text**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqx3emDKV3Zr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "aligned_fr = \"/content/drive/My Drive/ift6759/ift6759-t6-p2/data/aligned_formated_fr\"\n",
        "aligned_en = \"/content/drive/My Drive/ift6759/ift6759-t6-p2/data/aligned_unformated_en\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BtJKdhhGkjEP",
        "colab_type": "text"
      },
      "source": [
        "**original unaligned**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P-MFthyqkimk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "original_unaligned_fr = \"/content/drive/My Drive/ift6759/ift6759-t6-p2/data/unaligned_formated_fr\"\n",
        "original_unaligned_en = \"/content/drive/My Drive/ift6759/ift6759-t6-p2/data/unaligned_unformated_en\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hPsVpMccWy1j",
        "colab_type": "text"
      },
      "source": [
        "**virtual text**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BsO58qi5WLU_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "unaligned_fr_1 = \"/content/drive/My Drive/ift6759/ift6759-t6-p2/data/cycle5/unaligned_formated_fr\"\n",
        "unaligned_en_1 = \"/content/drive/My Drive/ift6759/ift6759-t6-p2/data/cycle5/unaligned_unformated_en\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wcHteua3uR0T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Used only at final cycle\n",
        "unaligned_fr_2 = \"/content/drive/My Drive/ift6759/ift6759-t6-p2/data/cycle6/unaligned_formated_fr\"\n",
        "unaligned_en_2 = \"/content/drive/My Drive/ift6759/ift6759-t6-p2/data/cycle6/unaligned_unformated_en\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iOnNeQULW4NC",
        "colab_type": "text"
      },
      "source": [
        "**real dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kLpPZX6_mevi",
        "colab_type": "code",
        "outputId": "d8d63640-80f3-42ad-af47-19a200154db4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "encoder_dataset, encoder_v2id, encoder_id2v = encoder_preprocess(data=aligned_en)\n",
        "decoder_dataset, decoder_v2id, decoder_id2v = decoder_preprocess(data=aligned_fr)"
      ],
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 11000/11000 [00:09<00:00, 1130.07it/s]\n",
            "100%|██████████| 11000/11000 [00:48<00:00, 226.88it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kOzFMywhXAUK",
        "colab_type": "text"
      },
      "source": [
        "**virtual dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36_cgFMKXCX-",
        "colab_type": "code",
        "outputId": "cc461c0a-16a3-44ac-d2fb-c5a656e595ce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "encoder_dataset_1, encoder_v2id, encoder_id2v = encoder_preprocess(data=unaligned_en_1)\n",
        "decoder_dataset_1, decoder_v2id, decoder_id2v = decoder_preprocess(data=unaligned_fr_1)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 51200/51200 [00:42<00:00, 1194.15it/s]\n",
            "100%|██████████| 51200/51200 [00:01<00:00, 30318.91it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GD2WkWYGuuyn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "01b328c4-c396-43bd-ad7c-6836a738147d"
      },
      "source": [
        "# Used only at final cycle\n",
        "encoder_dataset_2, _, _ = encoder_preprocess(data=unaligned_en_2)\n",
        "decoder_dataset_2, _, _ = decoder_preprocess(data=unaligned_fr_2)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 51200/51200 [00:06<00:00, 8341.68it/s] \n",
            "100%|██████████| 51200/51200 [02:26<00:00, 350.64it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGSX3q-8u4tp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoder_dataset = np.concatenate((encoder_dataset_1, encoder_dataset_2), 0)\n",
        "decoder_dataset = np.concatenate((decoder_dataset_1, decoder_dataset_2), 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PnXJ0hwbXV2d",
        "colab_type": "text"
      },
      "source": [
        "#### Split Train-Valid\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G5O3Lg1XKNia",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "(\n",
        "    input_tensor_train,\n",
        "    input_tensor_valid,\n",
        "    target_tensor_train,\n",
        "    target_tensor_valid\n",
        ") = train_test_split(encoder_dataset, decoder_dataset, test_size=0.2, shuffle=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YkYCU30nONMd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BUFFER_SIZE = len(input_tensor_train)\n",
        "batch_size = 32\n",
        "dataset_train = tf.data.Dataset.from_tensor_slices(\n",
        "    (input_tensor_train, target_tensor_train)).batch(batch_size, drop_remainder=False)\n",
        "dataset_valid = tf.data.Dataset.from_tensor_slices(\n",
        "    (input_tensor_valid, target_tensor_valid)).batch(batch_size, drop_remainder=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P9-BgT6v_wyc",
        "colab_type": "text"
      },
      "source": [
        "#### View data sample"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "otUIR4qqKp2W",
        "colab_type": "code",
        "outputId": "d87c077e-1873-4974-f091-60f12ff21ff5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "source": [
        "for i,j in dataset_train:\n",
        "  for z in i[10].numpy():\n",
        "    print(encoder_id2v[z], end = ' ')\n",
        "  print(\"\\n\")\n",
        "  for z in j[10].numpy():\n",
        "    print(decoder_id2v[z], end = ' ')\n",
        "  print(\"\\n\")\n",
        "  break\n",
        "\n",
        "for i,j in dataset_valid:\n",
        "  for z in i[0].numpy():\n",
        "    print(encoder_id2v[z], end = ' ')\n",
        "  print(\"\\n\")\n",
        "  for z in j[0].numpy():\n",
        "    print(decoder_id2v[z], end = ' ')\n",
        "  print(\"\\n\")\n",
        "  break\n"
      ],
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<SOS> regulation applying a scheme of generalised tariff preferences vote <EOS> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> \n",
            "\n",
            "<SOS> <CAP> règlement appliquant un schéma de préférences tarifaires généralisées <UNK> vote <UNK> <EOS> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> \n",
            "\n",
            "<SOS> my wish is for you to help a strong sustainable movement to educate every child about food to inspire families to cook again and to empower people everywhere to fight obesity <EOS> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> \n",
            "\n",
            "<SOS> <CAP> mon souhait est que vous <UNK> un puissant mouvement durable pour éduquer chaque enfant à l <UNK> alimentation <UNK> pour inspirer les familles à cuisiner à nouveau <UNK> et dynamiser les gens partout à lutter contre l <UNK> obésité <UNK> <EOS> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "J0Qjg6vuaHNt"
      },
      "source": [
        "# Transformer model for language understanding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "BSiBsXrV-oIa"
      },
      "source": [
        "## Positional encoding\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WhIOZjMNKujn",
        "colab": {}
      },
      "source": [
        "def get_angles(pos, i, d_model):\n",
        "  angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "  return pos * angle_rates"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "1Rz82wEs5biZ",
        "colab": {}
      },
      "source": [
        "def positional_encoding(position, d_model):\n",
        "  angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
        "                          np.arange(d_model)[np.newaxis, :],\n",
        "                          d_model)\n",
        "  \n",
        "  # apply sin to even indices in the array; 2i\n",
        "  angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "  \n",
        "  # apply cos to odd indices in the array; 2i+1\n",
        "  angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "    \n",
        "  pos_encoding = angle_rads[np.newaxis, ...]\n",
        "    \n",
        "  return tf.cast(pos_encoding, dtype=tf.float32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "a_b4ou4TYqUN"
      },
      "source": [
        "## Masking"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "U2i8-e1s8ti9",
        "colab": {}
      },
      "source": [
        "def create_padding_mask(seq):\n",
        "  seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "  \n",
        "  # add extra dimensions to add the padding\n",
        "  # to the attention logits.\n",
        "  return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "dVxS8OPI9uI0",
        "colab": {}
      },
      "source": [
        "def create_look_ahead_mask(size):\n",
        "  mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
        "  return mask  # (seq_len, seq_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "xluDl5cXYy4y"
      },
      "source": [
        "## Scaled dot product attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "LazzUq3bJ5SH",
        "colab": {}
      },
      "source": [
        "def scaled_dot_product_attention(q, k, v, mask):\n",
        "  \"\"\"Calculate the attention weights.\n",
        "  q, k, v must have matching leading dimensions.\n",
        "  k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n",
        "  The mask has different shapes depending on its type(padding or look ahead) \n",
        "  but it must be broadcastable for addition.\n",
        "  \n",
        "  Args:\n",
        "    q: query shape == (..., seq_len_q, depth)\n",
        "    k: key shape == (..., seq_len_k, depth)\n",
        "    v: value shape == (..., seq_len_v, depth_v)\n",
        "    mask: Float tensor with shape broadcastable \n",
        "          to (..., seq_len_q, seq_len_k). Defaults to None.\n",
        "    \n",
        "  Returns:\n",
        "    output, attention_weights\n",
        "  \"\"\"\n",
        "\n",
        "  matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n",
        "  \n",
        "  # scale matmul_qk\n",
        "  dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
        "  scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
        "\n",
        "  # add the mask to the scaled tensor.\n",
        "  if mask is not None:\n",
        "    scaled_attention_logits += (mask * -1e9)  \n",
        "\n",
        "  # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
        "  # add up to 1.\n",
        "  attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n",
        "\n",
        "  output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n",
        "\n",
        "  return output, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "n90YjClyInFy",
        "colab": {}
      },
      "source": [
        "def print_out(q, k, v):\n",
        "  temp_out, temp_attn = scaled_dot_product_attention(\n",
        "      q, k, v, None)\n",
        "  print ('Attention weights are:')\n",
        "  print (temp_attn)\n",
        "  print ('Output is:')\n",
        "  print (temp_out)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "kmzGPEy64qmA"
      },
      "source": [
        "## Multi-head attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "BSV3PPKsYecw",
        "colab": {}
      },
      "source": [
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads):\n",
        "    super(MultiHeadAttention, self).__init__()\n",
        "    self.num_heads = num_heads\n",
        "    self.d_model = d_model\n",
        "    \n",
        "    assert d_model % self.num_heads == 0\n",
        "    \n",
        "    self.depth = d_model // self.num_heads\n",
        "    \n",
        "    self.wq = tf.keras.layers.Dense(d_model)\n",
        "    self.wk = tf.keras.layers.Dense(d_model)\n",
        "    self.wv = tf.keras.layers.Dense(d_model)\n",
        "    \n",
        "    self.dense = tf.keras.layers.Dense(d_model)\n",
        "        \n",
        "  def split_heads(self, x, batch_size):\n",
        "    \"\"\"Split the last dimension into (num_heads, depth).\n",
        "    Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
        "    \"\"\"\n",
        "    x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
        "    return tf.transpose(x, perm=[0, 2, 1, 3])\n",
        "    \n",
        "  def call(self, v, k, q, mask):\n",
        "    batch_size = tf.shape(q)[0]\n",
        "    \n",
        "    q = self.wq(q)  # (batch_size, seq_len, d_model)\n",
        "    k = self.wk(k)  # (batch_size, seq_len, d_model)\n",
        "    v = self.wv(v)  # (batch_size, seq_len, d_model)\n",
        "    \n",
        "    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
        "    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
        "    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
        "    \n",
        "    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
        "    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "    scaled_attention, attention_weights = scaled_dot_product_attention(\n",
        "        q, k, v, mask)\n",
        "    \n",
        "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n",
        "\n",
        "    concat_attention = tf.reshape(scaled_attention, \n",
        "                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
        "        \n",
        "    return output, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RdDqGayx67vv"
      },
      "source": [
        "## Point wise feed forward network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ET7xLt0yCT6Z",
        "colab": {}
      },
      "source": [
        "def point_wise_feed_forward_network(d_model, dff):\n",
        "  return tf.keras.Sequential([\n",
        "      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n",
        "      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n",
        "  ])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "7e7hKcxn6-zd"
      },
      "source": [
        "## Encoder and decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "QFv-FNYUmvpn"
      },
      "source": [
        "### Encoder layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ncyS-Ms3i2x_",
        "colab": {}
      },
      "source": [
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "    super(EncoderLayer, self).__init__()\n",
        "\n",
        "    self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "\n",
        "    self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    \n",
        "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "  def call(self, x, training, mask):\n",
        "\n",
        "    attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n",
        "    attn_output = self.dropout1(attn_output, training=training)\n",
        "    out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n",
        "    \n",
        "    ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n",
        "    ffn_output = self.dropout2(ffn_output, training=training)\n",
        "    out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n",
        "    \n",
        "    return out2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "6LO_48Owmx_o"
      },
      "source": [
        "### Decoder layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9SoX0-vd1hue",
        "colab": {}
      },
      "source": [
        "class DecoderLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "    super(DecoderLayer, self).__init__()\n",
        "\n",
        "    self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
        "    self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
        "\n",
        "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        " \n",
        "    self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    \n",
        "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout3 = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "    \n",
        "  def call(self, x, enc_output, training, \n",
        "           look_ahead_mask, padding_mask):\n",
        "    # enc_output.shape == (batch_size, input_seq_len, d_model)\n",
        "\n",
        "    attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n",
        "    attn1 = self.dropout1(attn1, training=training)\n",
        "    out1 = self.layernorm1(attn1 + x)\n",
        "    \n",
        "    attn2, attn_weights_block2 = self.mha2(\n",
        "        enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n",
        "    attn2 = self.dropout2(attn2, training=training)\n",
        "    out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n",
        "    \n",
        "    ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n",
        "    ffn_output = self.dropout3(ffn_output, training=training)\n",
        "    out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n",
        "    \n",
        "    return out3, attn_weights_block1, attn_weights_block2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "SE1H51Ajm0q1"
      },
      "source": [
        "### Encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "jpEox7gJ8FCI",
        "colab": {}
      },
      "source": [
        "class Encoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n",
        "               maximum_position_encoding, rate=0.1):\n",
        "    super(Encoder, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.num_layers = num_layers\n",
        "    \n",
        "    self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
        "    self.pos_encoding = positional_encoding(maximum_position_encoding, \n",
        "                                            self.d_model)\n",
        "    \n",
        "    \n",
        "    self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n",
        "                       for _ in range(num_layers)]\n",
        "  \n",
        "    self.dropout = tf.keras.layers.Dropout(rate)\n",
        "        \n",
        "  def call(self, x, training, mask):\n",
        "\n",
        "    seq_len = tf.shape(x)[1]\n",
        "    \n",
        "    # adding embedding and position encoding.\n",
        "    x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n",
        "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "    x += self.pos_encoding[:, :seq_len, :]\n",
        "\n",
        "    x = self.dropout(x, training=training)\n",
        "    \n",
        "    for i in range(self.num_layers):\n",
        "      x = self.enc_layers[i](x, training, mask)\n",
        "    \n",
        "    return x  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "p-uO6ls8m2O5"
      },
      "source": [
        "### Decoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "d5_d5-PLQXwY",
        "colab": {}
      },
      "source": [
        "class Decoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n",
        "               maximum_position_encoding, rate=0.1):\n",
        "    super(Decoder, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.num_layers = num_layers\n",
        "    \n",
        "    self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
        "    self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
        "    \n",
        "    self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n",
        "                       for _ in range(num_layers)]\n",
        "    self.dropout = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "  def call(self, x, enc_output, training, \n",
        "           look_ahead_mask, padding_mask):\n",
        "\n",
        "    seq_len = tf.shape(x)[1]\n",
        "    attention_weights = {}\n",
        "    \n",
        "    x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n",
        "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "    x += self.pos_encoding[:, :seq_len, :]\n",
        "    \n",
        "    x = self.dropout(x, training=training)\n",
        "\n",
        "    for i in range(self.num_layers):\n",
        "      x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n",
        "                                             look_ahead_mask, padding_mask)\n",
        "      \n",
        "      attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
        "      attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
        "    \n",
        "    # x.shape == (batch_size, target_seq_len, d_model)\n",
        "    return x, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "y54xnJnuYgJ7"
      },
      "source": [
        "## Create the Transformer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "PED3bIpOYkBu",
        "colab": {}
      },
      "source": [
        "class Transformer(tf.keras.Model):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n",
        "               target_vocab_size, pe_input, pe_target, rate=0.1):\n",
        "    super(Transformer, self).__init__()\n",
        "\n",
        "    self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n",
        "                           input_vocab_size, pe_input, rate)\n",
        "\n",
        "    self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n",
        "                           target_vocab_size, pe_target, rate)\n",
        "\n",
        "    self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
        "    \n",
        "  def call(self, inp, tar, training, enc_padding_mask, \n",
        "           look_ahead_mask, dec_padding_mask):\n",
        "\n",
        "    enc_output = self.encoder(inp, training, enc_padding_mask)  # (batch_size, inp_seq_len, d_model)\n",
        "    \n",
        "    # dec_output.shape == (batch_size, tar_seq_len, d_model)\n",
        "    dec_output, attention_weights = self.decoder(\n",
        "        tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
        "    \n",
        "    final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n",
        "    \n",
        "    return final_output, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wsINyf1VEQLC"
      },
      "source": [
        "## Set hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lnJn5SLA2ahP",
        "colab": {}
      },
      "source": [
        "num_layers = 2\n",
        "d_model = 128\n",
        "dff = 512\n",
        "num_heads = 16\n",
        "\n",
        "input_vocab_size = len(encoder_v2id) + 1\n",
        "target_vocab_size = len(decoder_v2id) + 1\n",
        "dropout_rate = 0.3"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "xYEGhEOtzn5W"
      },
      "source": [
        "## Optimizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iYQdOO1axwEI",
        "colab": {}
      },
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "  def __init__(self, d_model, warmup_steps=4000):\n",
        "    super(CustomSchedule, self).__init__()\n",
        "    \n",
        "    self.d_model = d_model\n",
        "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
        "\n",
        "    self.warmup_steps = warmup_steps\n",
        "    \n",
        "  def __call__(self, step):\n",
        "    arg1 = tf.math.rsqrt(step)\n",
        "    arg2 = step * (self.warmup_steps ** -1.5)\n",
        "    \n",
        "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7r4scdulztRx",
        "colab": {}
      },
      "source": [
        "learning_rate = CustomSchedule(d_model)\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n",
        "                                     epsilon=1e-9)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "f33ZCgvHpPdG",
        "outputId": "2a5a4eab-98ba-4461-f657-579796cbd5a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        }
      },
      "source": [
        "temp_learning_rate_schedule = CustomSchedule(d_model)\n",
        "\n",
        "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n",
        "plt.ylabel(\"Learning Rate\")\n",
        "plt.xlabel(\"Train Step\")"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, 'Train Step')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 136
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEGCAYAAACtqQjWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxV9Z34/9c7OwlkIQlhCRAIYQmKqBH3peKC2sq0xRHqd2qro9NWu3esfjvjOP7q/GrbqdZW67jgNipQaiu27nXfgLiggCC5Nwhhy02ASMISkry/f5xP4BJvkpvk3tyb3Pfz8cgj537OOZ/zvjeQd875fM77iKpijDHGREJSrAMwxhgzeFhSMcYYEzGWVIwxxkSMJRVjjDERY0nFGGNMxKTEOoBYKigo0JKSkliHYYwxA8q7775bp6qFodYldFIpKSmhsrIy1mEYY8yAIiKfdrbOLn8ZY4yJGEsqxhhjIsaSijHGmIixpGKMMSZiLKkYY4yJmKgmFRGZIyLrRaRKRK4PsT5dRBa79ctFpCRo3Q2ufb2InB/UvlBEakVkdSfH/LGIqIgUROM9GWOM6VzUkoqIJAN3AhcA5cACESnvsNmVwC5VnQTcBtzq9i0H5gPTgTnAXa4/gAddW6hjjgXOAzZF9M0YY4wJSzTPVGYBVarqV9VmYBEwt8M2c4GH3PJSYLaIiGtfpKoHVLUaqHL9oaqvATs7OeZtwHXAoKznr6osWbmZxgMtsQ7FGGNCimZSGQNsDnpd49pCbqOqLUADkB/mvkcQkbnAFlVd1c12V4tIpYhUBgKBcN5H3Phg826u+9OH/HTph7EOxRhjQhoUA/Uikgn8X+DG7rZV1XtUtUJVKwoLQ1YZiFubdu4F4IWPd8Q4EmOMCS2aSWULMDbodbFrC7mNiKQAOUB9mPsGKwUmAKtEZKPb/j0RGdmH+OOOL9AEQHNLG5tdgjHGmHgSzaSyEigTkQkikoY38L6swzbLgMvd8jzgJfWeb7wMmO9mh00AyoAVnR1IVT9S1RGqWqKqJXiXy45T1e2RfUux5Qs0IuItP7N6W2yDMcaYEKKWVNwYybXAc8DHwBJVXSMiN4vIxW6z+4F8EakCfgRc7/ZdAywB1gLPAteoaiuAiDwOvA1MEZEaEbkyWu8h3vgDTZw5uZDpo7N5ZvWgypfGmEEiqlWKVfVp4OkObTcGLe8HLulk31uAW0K0LwjjuCU9jTXetbUp1XWNnFKazwklw/nVc+vZ1rCPUTlDYh2aMcYcMigG6hPB1oZ97D/YxsTCLOYc5Q0VPWtnK8aYOGNJZYDwu0H60sKhlBYOZerIYTy1amuMozLGmCNZUhkgfIFGACYWZgEwd+YY3tu0m0/rm2IZljHGHMGSygDhDzQxLCOFwqHpAMydORoR+Mv7drZijIkfllQGCF+gkYmFQxE3p3h07hBOmpDPn9+vwZuFbYwxsWdJZYDwB5ooLcg6ou3Lx41hY/1e3t+8O0ZRGWPMkSypDACNB1rY/tl+SkcMPaL9gqNGkp6SxF/e76rYgDHG9B9LKgNAtZv5NbHDmcqwjFTOLS/iqVVbOdDSGovQjDHmCJZUBgB/nTfzq+OZCsAlFWPZtfcgz6+xIpPGmNizpDIA+GobSRIYn5/5uXWnTyqgOG8Ijy2355IZY2LPksoA4Ktrojgvk/SU5M+tS0oSFswax9v+evzuXhZjjIkVSyoDgK+2kdLCrE7XX1JRTEqSsGjl5k63McaY/mBJJc61tSkb65uYWPj58ZR2I4ZlcM60Ipa+W2MD9saYmLKkEufaC0mWdpFUAL524jh2NjVbkUljTExZUolz7U97nNjF5S+A0yYVMKEgi4VvbrQ77I0xMWNJJc61D753d6aSlCR889QSVm3ezXubdvVHaMYY8zmWVOKcL9DIsIwUCoamdbvtvOOLyRmSyn2vV/dDZMYY83mWVOKcP9B0RCHJrmSmpbBg1jieW7OdzTv39kN0xhhzJEsqcc4faOpyOnFHl58yniQRHnxrY/SCMsaYTkQ1qYjIHBFZLyJVInJ9iPXpIrLYrV8uIiVB625w7etF5Pyg9oUiUisiqzv09SsRWSciH4rIn0UkN5rvrT8cKiTZzXhKsFE5Q7jw6FEsXrmZhr0HoxidMcZ8XtSSiogkA3cCFwDlwAIRKe+w2ZXALlWdBNwG3Or2LQfmA9OBOcBdrj+AB11bRy8AR6nqDOAT4IaIvqEYqD70COHwz1QAvn1WKY0HWnjgLRtbMcb0r2ieqcwCqlTVr6rNwCJgbodt5gIPueWlwGzxBg/mAotU9YCqVgNVrj9U9TVgZ8eDqerzqtriXr4DFEf6DfW3w48QDv9MBWDaqGzOmVbEA29uZM9+O1sxxvSfaCaVMUBw3ZAa1xZyG5cQGoD8MPftyhXAM6FWiMjVIlIpIpWBQKAHXfY/f6DzQpLd+d7sSTTsO8gj73wahciMMSa0QTdQLyI/A1qAR0OtV9V7VLVCVSsKCwv7N7ge8gWaGDs8dCHJ7swozuXMyYXc93o1e5tbut/BGGMiIJpJZQswNuh1sWsLuY2IpAA5QH2Y+36OiHwD+CJwmQ6C28p9gcbPPZirJ7579iR2NjXz6DtWFt8Y0z+imVRWAmUiMkFE0vAG3pd12GYZcLlbnge85JLBMmC+mx02ASgDVnR1MBGZA1wHXKyqA/4mjbY2pbquqUczvzqqKBnOaZMK+MOrPhtbMcb0i6glFTdGci3wHPAxsERV14jIzSJysdvsfiBfRKqAHwHXu33XAEuAtcCzwDWq2gogIo8DbwNTRKRGRK50ff0eGAa8ICIfiMjd0Xpv/WHL7n0caGnr8SB9Rz+dM5WdTc3c+5o/QpEZY0znUqLZuao+DTzdoe3GoOX9wCWd7HsLcEuI9gWdbD+pT8HGGX9d76YTd3R0cQ4XzRjFfW9U808nl1A4LD0S4RljTEiDbqB+sPDV9m46cSg/OW8KzS1t/O6lDX3uyxhjumJJJU7568IvJNmdCQVZXHrCWB5bvomN7gzIGGOiwZJKnPJqfoVXSDIc359dRnpKEj//28cR6c8YY0KxpBKnfIHGbh/M1RMjsjP47uwyXvx4B6+sr41Yv8YYE8ySShxqPNDCjs8O9Gk6cSjfPLWECQVZ3PzUWppb2iLatzHGgCWVuHT4aY+RO1MBSE9J5sYvleOva+JBKzZpjIkCSypxyH/oufSRPVMB+MKUEcyeOoLfvriB7Q37I96/MSaxWVKJQ74+FJIMx41fKqdVlX9/cjWDoJqNMSaOWFKJQ/4+FJIMx/j8LH54zmReWLuDZ1Zvj8oxjDGJyZJKHPIFGiM+SN/RladN4Kgx2dz45Bp7QqQxJmIsqcSZ9kKSfalOHI6U5CRu/eoMdu1t5pan10b1WMaYxGFJJc60F5IsHRHdMxWA6aNzuPqMiSyprOFlu3fFGBMBllTizKFHCEf5TKXd92eXMaVoGNct/ZD6xgP9ckxjzOBlSSXORHM6cSgZqcncPn8mDXsPcsMTH9lsMGNMn1hSiTP+ukayI1RIMlzTRmVz3ZwpPL92B0sqN/fbcY0xg48llTjjq21iYgQLSYbrilMncEppPv/51NpDd/QbY0xPWVKJM/666E8nDiUpSfjvfzyG9JQkvvPoe+xrbu33GIwxA58llTiyZ/9Bdnx2IKLViXtiVM4Qbrt0Jut37OHf/mJ32xtjes6SShypjtAjhPvirCkj+O7ZZfzpvRoWr7TxFWNMz0Q1qYjIHBFZLyJVInJ9iPXpIrLYrV8uIiVB625w7etF5Pyg9oUiUisiqzv0NVxEXhCRDe57XjTfWzT4DlUn7v/LX8G+P7uM08sKuHHZGlZvaYhpLMaYgSVqSUVEkoE7gQuAcmCBiJR32OxKYJeqTgJuA251+5YD84HpwBzgLtcfwIOuraPrgb+rahnwd/d6QPEHmkgSGBelQpLhSk4Sbr90JgVZaVz1cCW1e6yasTEmPNE8U5kFVKmqX1WbgUXA3A7bzAUecstLgdniTXuaCyxS1QOqWg1Uuf5Q1deAnSGOF9zXQ8A/RPLN9Ad/oIlxUSwk2RP5Q9O59/IKdu89yNUPv8v+gzZwb4zpXjSTyhgg+KJ8jWsLuY2qtgANQH6Y+3ZUpKrb3PJ2oCjURiJytYhUikhlIBAI5330G+8RwrG99BVs+ugcbp8/kw827+a6pR/awL0xpluDcqBevd9+IX8Dquo9qlqhqhWFhYX9HFnnWl0hyVgO0ody/vSRXDdnCstWbeV3L1XFOhxjTJyLZlLZAowNel3s2kJuIyIpQA5QH+a+He0QkVGur1HAgKqQuNUVkoynM5V23z6zlK8cN4bfvPAJS2xGmDGmC9FMKiuBMhGZICJpeAPvyzpsswy43C3PA15yZxnLgPludtgEoAxY0c3xgvu6HHgyAu+h3/R3IcmeEBF+8ZUZnDG5kOuf+JAX1u6IdUjGmDgVtaTixkiuBZ4DPgaWqOoaEblZRC52m90P5ItIFfAj3IwtVV0DLAHWAs8C16hqK4CIPA68DUwRkRoRudL19QvgXBHZAJzjXg8Y7YUk+6PkfW+kpSTxh8uO4+jiXK597D1WVIeaK2GMSXSSyIOvFRUVWllZGeswAPjZnz/iqVVbWfUf5/V73a+e2NnUzLy73yKw5wCLrz6Z8tHZsQ7JGNPPRORdVa0ItW5QDtQPRP5AE6Uj+r+QZE8Nz0rj4StmMTQ9hcvue4ePt30W65CMMXHEkkqc8AUamVgQn5e+OirOy+Txq04iPSWZy+5bzvrte2IdkjEmTlhSiQN79h+kdk/sCkn2RklBFo9ffRKpycLX7n2HDTsssRhjLKnEhUOD9HE4nbgrEwqyeOyqk0hOEhbca5fCjDGWVOKCv669kOTAOVNpV1o4lMevPomUpCQu/Z+3efdTmxVmTCLrNqmIyGQR+Xt7VWARmSEi/xb90BKHP9BEcpLEvJBkb5UWDmXpt08mf2g6l923nFfWD6j7To0xERTOmcq9wA3AQQBV/RDvRkYTIb5AI2PzhsRFIcneKs7LZMm/nMzEgqFc9XAlT63aGuuQjDExEE5SyVTVjnezt0QjmETlDzQNuPGUUAqHpbPoX07i2LF5fG/R+9zzms+KUBqTYMJJKnUiUoor0Cgi84BtXe9iwtXapvjrmgbUzK+uZGek8vCVs7jwqFH819Pr+L9/Xs3B1rZYh2WM6ScpYWxzDXAPMFVEtgDVwGVRjSqBbN29j+Y4LSTZWxmpyfxuwbGMz8/krld81Ozay52XHUd2RmqsQzPGRFk4ZyqqqucAhcBUVT0tzP1MGOLlEcKRlpQkXDdnKr+cN4O3ffV89a632FjXFOuwjDFRFk5y+BOAqjapavsdbkujF1Ji8bl7VAbL5a+O/rFiLA9fOYtA4wG+9Ps3+PvHVuHYmMGs06QiIlNF5KtAjoh8JejrG0BGv0U4yPkDjeQMSSU/Ky3WoUTNKaUFPHXtaYzPz+TKhyr5zfPraW2zAXxjBqOuxlSmAF8EcoEvBbXvAa6KZlCJxHuEcFbcF5Lsq7HDM1n6rVP497+s5o6XqlhV08Dtl84kbxAnU2MSUadJRVWfBJ4UkZNV9e1+jCmh+ANNnF4WP481jqaM1GR+OW8GM8flctOyNVx4x+vcdulMTpqYH+vQjDEREs6Yyvsico2I3CUiC9u/oh5ZAmgvJFk6YnCOp4QiIlx24nie+PapZKQms+Ded/jN8+tpsWnHxgwK4SSVR4CRwPnAq3jPi7eStBHQXkhyoJS8j6Sji3P463dP46vHFXPHS1Vces87bN65N9ZhGWP6KJykMklV/x1oUtWHgIuAE6MbVmJoLyQ5KYHOVIJlpafw60uO4Y4Fx/LJ9j1c+NvXWVK52e7CN2YACyepHHTfd4vIUUAOMCJ6ISUOX60rJDk8MZNKu4uPGc3T3z+daaOzuW7ph3zjgZVs3b0v1mEZY3ohnKRyj4jkAf8GLAPWArdGNaoE4a9rZNzwTNJS7F7SscMzWXTVSfznxdNZUb2T8297jcUrN9lZizEDTLe/zVT1PlXdpaqvqepEVR0BPBNO5yIyR0TWi0iViFwfYn26iCx265eLSEnQuhtc+3oROb+7PkVktoi8JyIfiMgbIjIpnBhjyVfbxMSCxD5LCZaUJFx+SgnP/eAMykdn89M/fcTXF66wO/GNGUC6TCoicrKIzBOREe71DBF5DHizu45FJBm4E7gAKAcWiEh5h82uBHap6iTgNtwZkNtuPjAdmAPcJSLJ3fT5B+AyVZ0JPIZ3ZhW3WtuU6vrBU0gyksblZ/L4VSdx89zpvL9pN+fd/hq/fXEDB1paYx2aMaYbXd1R/ytgIfBV4G8i8nPgeWA5UBZG37OAKlX1q2ozsAiY22GbucBDbnkpMFu8uwDnAotU9YCqVgNVrr+u+lQg2y3nAHH9QI/2QpKDreZXpCQlCV8/uYS///hMzisv4rYXP2HO7a/zxoa6WIdmjOlCV3fUXwQcq6r73ZjKZuAoVd0YZt9j3D7tavj8rLFD26hqi4g0APmu/Z0O+45xy531+c/A0yKyD/gMOClUUCJyNXA1wLhx48J8K5FX5QpJDqbqxNFQlJ3B7792HP9YEeDGJ1fzf+5fzhdnjOKGC6cxJndIrMMzxnTQ1eWv/aq6H0BVdwEbepBQYuGHwIWqWgw8APwm1Eaqeo+qVqhqRWFh7O5kb79HZSA+lz4WzphcyLM/OIMfnFPGC2t3cPavX+FXz62j8YA9L86YeNLVmcpEEVkW9HpC8GtVvbibvrcAY4NeF7u2UNvUiEgK3mWr+m72/Vy7iBQCx6jqcte+GHi2m/hiyucKSQ632ldhy0hN5gfnTOaSirH86tl13PmyjyWVNfzkvMnMO34syUmDu36aMQNBV0ml4/jHf/ew75VAmYhMwEsI84GvddhmGXA58DYwD3hJVdUlr8dE5DfAaLwxnBWAdNLnLrxqypNV9RPgXODjHsbbr/wJUkgyGsbkDuH2+cdy+Skl/PxvH/PTP33EA29u5PoLpnLm5EL7TI2Joa4KSr7al47dGMm1wHNAMrBQVdeIyM1ApaouA+4HHhGRKmAnXpLAbbcE756YFuAaVW0FCNWna78K+JOItOElmSv6En+0+QJNnDk5MQpJRsux4/JY+q2Tefqj7fzi2Y/5xgMrOaEkj5+cN4UTrUilMTEhiXxzWUVFhVZWVvb7cffsP8jRNz3PdXOm8J2z4v52mgGhuaWNxZWb+f1LG9jx2QFOLyvgJ+dN4ZixubEOzZhBR0TeVdWKUOvsVu4YODxIbzO/IiUtJYl/Omk8r/7rF/jZhdNYs/Uz5t75Jlc9XMmHNbtjHZ4xCaOrMRUTJYefS28zvyItIzWZq86YyIITx7HwjWrufd3PC2t3cHpZAdd8YRInThhuYy7GRFG3SUVEnsK7sTBYA1AJ/E/7tGMTPn/ACklG29D0FL43u4xvnlrC/76zifvf8DP/nnc4fnwe13yhlC9MGWHJxZgoCOfylx9oBO51X5/hPU9lsnttesgXsEKS/WVYRirfPquUN356NjfPnc72hv1c8WAlF/z2df78fg3NLfZwMGMiKZzLX6eo6glBr58SkZWqeoKIrIlWYIOZP2CFJPtbRmoyXz+5hAWzxvHkB1v5wytV/HDxKv7r6XV8/aTxfO3EceQPTY91mMYMeOH8qTxURA7VM3HL7SPMzVGJahBrLyRZOsIG6WMhNTmJeccX88IPz+TBb57AtFHZ/PcLn3DyL17ip0s/ZN32z2IdojEDWjhnKj8G3hARH97NhxOA74hIFoeLQZowbdnlFZK0M5XYSkoSzpoygrOmjGDDjj088NZGnnivhsWVmzmlNJ//c9J4zi0vIjXZLlEa0xPdJhVVfVpEyoCprml90OD87VGLbJDyuUcI25lK/CgrGsZ/fflo/vW8KTy+chP/+/anfOfR9ygYms4/VhSzYNY4xg7PjHWYxgwI4U4pPh4ocdsfIyKo6sNRi2oQ89W66sR2phJ38rLS+M5Zk/iXM0p59ZNaHlu+ibtf9fGHV32cXlbI12aNY/a0EXb2YkwXwplS/AhQCnwAtD8lSQFLKr3gr2siN9MKScaz5CTh7KlFnD21iK2797F45WYWr9zMt/73XQqHpfMPM0fzleOKmTYqu/vOjEkw4ZypVADlmsj1XCLIV9vIxAIrJDlQjM4dwg/Pncx3z57Ey+sD/LFyMw++tZF7X6+mfFQ2XzluDHNnjqFwmM0cMwbCSyqrgZHAtijHkhD8dVZIciBKSU7i3PIizi0vYmdTM0+t2soT79Xw8799zP//zDrOnFzIV44bwznTishITY51uMbETDhJpQBYKyIrgAPtjWE8T8V08Nn+gwT2HLCaXwPc8Kw0Lj+lhMtPKWHDjj088f4W/vzeFl5aV0tWWjLnlBdx0dGjOHNKIekplmBMYgknqdwU7SASRXshyYlW82vQKCsaxk/nTOUn503hHX89f/1wK8+s3s6TH2xlWHoK504v4oszRnHapEKroGASQjhTivv0XBVzmP9QIUk7UxlskpOEUycVcOqkAm6eexRv+er566qtPLdmO0+8t4XsjBTOnz6SC44eySmlBXaJzAxanSYVEXlDVU8TkT0cWVBSAFVVm/rSQ75Aoyskafc8DGapyUmcObmQMycXcsuXj+aNqgB/XbWNZ1Zv54/v1pCZlsyZkws5t7yIs6eOIDfTZgKawaOrJz+e5r4P679wBjd/oMkKSSaYtJSkQ9OTD7S08ravnufX7uDFtTt4ZvV2kpOEWSXDD00CsJsszUAX1pMfRSQZKCIoCanqpijG1S/6+8mP5932KuOGZ3Lf5Sd0v7EZ1NralA+3NPDC2u08v2YHG9xNsVNHDnPlYwo5fnye3Whp4lJXT34M5+bH7wL/AewA2uuEKzAjYhEmgNY2ZWP9Xs6aMiLWoZg4kJQkzByby8yxufzr+VPZWNfEC2t38OLHO7jvdT93v+pjaHoKp07K56wpIzhzciGjc4fEOmxjuhXO7K/vA1NUtb6nnYvIHOC3QDJwn6r+osP6dLw7848H6oFLVXWjW3cDcCXeXfzfU9XnuupTvLsJfw5c4vb5g6re0dOYo6W9kKQ97dGEUlKQxVVnTOSqMyayZ/9B3qyq59VPAry6vpbn1uwAYHLRUM6aMoIzygqpKMmzwX4Tl8JJKpvxnvTYI+6S2Z3AuUANsFJElqnq2qDNrgR2qeokEZkP3ApcKiLlwHxgOjAaeFFEJrt9OuvzG8BYYKqqtolIXJ0StD9CeKLN/DLdGJaRypyjRjLnqJGoKhtqG3llfS2vfhLggTeruec1P2kpSVSMz+OU0nxOmVTAjDE5pNilMhMHwkkqfuAVEfkbR978+Jtu9psFVKmqH0BEFgFzgeCkMpfD98EsBX7vzjjmAotU9QBQLSJVrj+66PPbwNdUtc3FVxvGe+s3PptObHpBRJhcNIzJRcO4+oxSmg608I6/nrd83tevn/8Env+EoekpnDhhOCeX5nPqpAKmFA0jKclKAZn+F05S2eS+0txXuMbgneW0qwFO7GwbVW0RkQYg37W/02HfMW65sz5L8c5yvgwE8C6ZbegYlIhcDVwNMG7cuI6ro8YXsEKSpu+y0lOYPa2I2dOKANjZ1Mzbvnre8tXxlq+ev6/z/pbKz0rjxInDOaHE+5o2KptkSzKmH3SZVNwlrMmqelk/xdMX6cB+Va0Qka8AC4HTO26kqvcA94A3+6u/gvMHGq3cvYm44VlpXDRjFBfNGAXA1t37eNtXz5u+Opb7d/L0R9sBGJqewnHj85hVkscJJcM5ZmyujcmYqOgyqahqq4iMF5E0Ve3po4O34I1xtCt2baG2qRGRFCAHb8C+q307a68BnnDLfwYe6GG8UeWva+IsKyRpomx07hC+enwxXz2+GPCSzMqNO72v6l3e5TIgLTmJGcU5VJQMZ9aEPGaOzbOzaBMR4Y6pvCkiy4Cm9sYwxlRWAmUiMgHvF/984GsdtlkGXA68DcwDXlJVdcd6TER+gzdQXwaswLubv7M+/wJ8AagGzgQ+CeO99Yv2QpI2SG/62+jcIcyd6ZXnB9i9t5nKjbtYuXEnKzbudNOXvRP2kvxMZo7N5dhxecwcm8u0Udl2o67psXCSis99JQFh313vxkiuBZ7Dm/67UFXXiMjNQKWqLgPuBx5xA/E78ZIEbrsleAPwLcA1qtoKEKpPd8hfAI+KyA+BRuCfw4012toLSdp0YhNruZlpnFNexDnl3pjMvuZWVtXs5oPNu/lg027e8tXzlw+2Al41gKPH5LhE491TMyZ3iD0LyHQprDvqB6v+uqP+T+/W8OM/ruLFH53JJHs2vYljqsq2hv18sHk372/axfubdvPRlgYOtHj3PRcOS2fGmByOcl9Hj8mhKDvdEk2C6esd9YXAdXj3jGS0t6vq2RGLcJDz11khSTMwiAijc4cwOncIFx7tDf4fbG1j3bY9vL95Fx+4JPPy+lra3N+jBUPTOWpMNkcHJZpRORmWaBJUOJe/HgUWA18EvoU3BhKIZlCDja+2ifFWSNIMUKnJSRxdnMPRxTl8/WSvbW9zC2u3fsbqLQ18tMX7/tongUOJJj8rjeljcjh6TDbTRmUzdWQ2JfmZdoNmAggnqeSr6v0i8n33bJVXRWRltAMbTPx1jfZgLjOoZKalUFEynIqS4Yfa9jW38vF2l2hqGvhoSwN3V9XR6jJNekoSk4uGMXXkMC/RjBrGtJHZ5Nmss0ElnKRy0H3fJiIXAVuB4V1sb4K0tikb6/byBSskaQa5IWnJHDcuj+PG5R1q23+wlaraRtZt38O6bZ+xbvseXlpXyx/frTm0TVF2OlNHHk4yU0cNo7RwqFVoHqDCSSo/F5Ec4MfA74Bs4IdRjWoQqdm1l+bWNjtTMQkpIzX50KB+sMCeA6zb/hnrtu3hY/f9bV89za3ehIDUZGFCQRZlI4ZROmIoZSOGUlY0lAkFWaSn2E2b8Sycxwn/1S024N0HYnrg8HRim/VlTLvCYekUDivk9LLDNwQfbG2juq6Jj90ZzYYdjazd9hnPrN52aKwmSWB8fhaTghLNpMJhlI7IIjMtnL+RTcx/dBMAABPjSURBVLSFM/trMvAHoEhVjxKRGcDFqvrzqEc3CFh1YmPCk5qcdKh45tyg9v0HW6mua2JDbSNVO/awobaRDbWNvLyulpa2w7dEFOcNoWzEUEoLhzKhMIsJBVlMLBhqU577WTip/V7gX4H/AVDVD0XkMbxnl5huWCFJY/omIzWZaaO8WWTBDra28Wl9Ext2NB5KNBt27OEtX/2h+2oAMtOSKcnPYkJhFhMLvGTTnnByMlP7++0MeuEklUxVXdEh07dEKZ5Bxx9otEtfxkRBanISk0YMY9KIYVwQ1N7Wpmz7bD/VgSaq6xrx1zVRXdfE6i0NPPPR4Utp4BXknBCUaCYUZDFueCbj8jPJzrCE0xvhJJU6ESnFe4QwIjIP2BbVqAYRX6CJL0yxQpLG9JekJGFM7hDG5A7htLKCI9Y1t7Sxaedequu8hFPtEs7rGwIsDZqRBpCbmcr44ZmMHZ7J+PxMxh1azmJkdoY9SqAT4SSVa/BKxU8VkS14BRsHQin8mGvYd5C6xgOUWmkWY+JCWkoSk0YMdeWSio5Y13ighU31e9m0s4lNO/fyaf1eNu3cy0dbGnh29fYjxm/SkpMozhtyRMJpP8MZkzuEYQl8lhPO7C8/cI6IZAFJqrpHRH4A3B716AY4f/sgvT1HxZi4NzQ9hfLR2ZSPzv7cupbWNrY17D8i2bQnn/c27WLP/iNHBLIzUijOy2RMnnfGVJznfY3J9dryMlMH7eSBsOfgqWpT0MsfYUmlW+3TiW3mlzEDW0pyEmPd5a9TJx25TlVp2HfwULLZsnsfW3btY8vufWyq38tbVXU0NbcesU9mWrJ3ia5DsinOG0Jx7hAKhqYP2MdB93Zi98B8t/3MF2gkJUkYn2+FJI0ZrESE3Mw0cjPTOGZs7ufWtyedml37qHHJxks6e6nZtY8PNu9m996DR+yTlpzEyJwMRuZkMDong5E5Qxh16PUQRuZkkJ+VFpeJp7dJJXHr5feAP9DEuOGZVm7CmAQWnHQ6VhZo13igha2791Gzay9bdu2jZvc+tjfsZ9vu/by7aRfbG7ZxsPXIX7upyUJR9uEk0550RrkENConIyZnPJ0mFRHZQ+jkIcCQqEU0iHiFJO3SlzGma0PTUw7d+BlKW5tS39TsJZqGfWz/bD9bd+9ne8O+Q8+/eXb1/kNlbtqlJHmJZ2ROBkXZ6d5ydgZF2RmcUprPiOyMkMfri06TiqqG/ZRH83lWSNIYEylJSeJK26RzdHHosx1VZWdTM9sa9rOt4XDC8Zb3s277Hl5dHzg0vvPwFbP6N6mYvmkvJGk3Phpj+oOIkD80nfyh6Z1eZgPYs/8gOz47wKicyCcUsKQSNYdrftl0YmNM/BiWkRrV+2iiOoIsInNEZL2IVInI9SHWp4vIYrd+uYiUBK27wbWvF5Hze9DnHSLSGK33FC6bTmyMSURRSyoikgzcCVwAlAMLRKS8w2ZXArtUdRJwG3Cr27ccmA9MB+YAd4lIcnd9ikgFkEcc8AWayLNCksaYBBPNM5VZQJWq+lW1GVgER1S0xr1+yC0vBWaLd5vpXGCRqh5Q1WqgyvXXaZ8u4fwKuC6K7ylsvoDN/DLGJJ5oJpUxwOag1zWuLeQ2qtqC9yCw/C727arPa4FlqtplsUsRuVpEKkWkMhAI9OgN9YQ/0ESpjacYYxLMoLgrT0RGA5fgPe64S6p6j6pWqGpFYWF0qge3F5K0MxVjTKKJZlLZAowNel3s2kJuIyIpQA5Q38W+nbUfC0wCqkRkI5ApIlWReiM9ZYUkjTGJKppJZSVQJiITRCQNb+B9WYdtlgGXu+V5wEuqqq59vpsdNgEoA1Z01qeq/k1VR6pqiaqWAHvd4H9M+NqfS28l740xCSZq96moaouIXAs8ByQDC1V1jYjcDFSq6jLgfuARd1axEy9J4LZbAqzFe8rkNaraChCqz2i9h97yu0KS44ZbIUljTGKJ6s2Pqvo08HSHthuDlvfjjYWE2vcW4JZw+gyxTUxPEfyBJsblWyFJY0zisd96UeALNDKxwC59GWMSjyWVCGtpbePT+r2UjrBBemNM4rGkEmE1u/Z5hSTtTMUYk4AsqUSYv84KSRpjEpcllQhrLyRpJe+NMYnIkkqE+QKN5GWmkmeFJI0xCciSSoT5Ak12lmKMSViWVCLMH2i08RRjTMKypBJBDXsPUtfYbIUkjTEJy5JKBPnczC+7/GWMSVSWVCLo8COE7fKXMSYxWVKJICskaYxJdJZUIsgXaLRCksaYhGa//SLIb9OJjTEJzpJKhLS0trGxvsnGU4wxCc2SSoTU7NrHwVa1QpLGmIRmSSVC2gtJWsl7Y0wis6QSIb5aN53YzlSMMQnMkkqE+OsaGZ6VZoUkjTEJLapJRUTmiMh6EakSketDrE8XkcVu/XIRKQlad4NrXy8i53fXp4g86tpXi8hCEUmN5nvryFfbxMQCu/RljElsUUsqIpIM3AlcAJQDC0SkvMNmVwK7VHUScBtwq9u3HJgPTAfmAHeJSHI3fT4KTAWOBoYA/xyt9xaKv84KSRpjTDTPVGYBVarqV9VmYBEwt8M2c4GH3PJSYLaIiGtfpKoHVLUaqHL9ddqnqj6tDrACKI7ieztCeyFJu0fFGJPooplUxgCbg17XuLaQ26hqC9AA5Hexb7d9uste/wQ82+d3ECbfoUcIW1IxxiS2wThQfxfwmqq+HmqliFwtIpUiUhkIBCJywMOPELbLX8aYxBbNpLIFGBv0uti1hdxGRFKAHKC+i3277FNE/gMoBH7UWVCqeo+qVqhqRWFhYQ/fUmg+V0hyrBWSNMYkuGgmlZVAmYhMEJE0vIH3ZR22WQZc7pbnAS+5MZFlwHw3O2wCUIY3TtJpnyLyz8D5wAJVbYvi+/ocf6CR8VZI0hhjSIlWx6raIiLXAs8BycBCVV0jIjcDlaq6DLgfeEREqoCdeEkCt90SYC3QAlyjqq0Aofp0h7wb+BR42xvr5wlVvTla7y+YL9Bk4ynGGEMUkwp4M7KApzu03Ri0vB+4pJN9bwFuCadP1x7V99KZltY2Pq1vYva0EbE4vDHGxBW7XtNHhwpJ2pmKMcZYUukrX6D9ufQ288sYYyyp9NGh59JbIUljjLGk0le+gBWSNMaYdpZU+sgfsEKSxhjTzpJKH/kCjTZIb4wxjiWVPmjYe5D6pmarTmyMMY4llT5oLyRpZyrGGOOxpNIHvtr26sR2pmKMMWBJpU/8dU2kJlshSWOMaWdJpQ98tY2MG26FJI0xpp39NuwDf50VkjTGmGCWVHqpvZCkDdIbY8xhllR6abMrJGmD9MYYc5gllV7yB2w6sTHGdGRJpZesOrExxnyeJZVe8geayM9KIzfTCkkaY0w7Syq95As02niKMcZ0YEmll7zqxDaeYowxwSyp9MLuvc3UNzVTOsLOVIwxJlhUk4qIzBGR9SJSJSLXh1ifLiKL3frlIlIStO4G175eRM7vrk8RmeD6qHJ9Rm2ww2dPezTGmJCillREJBm4E7gAKAcWiEh5h82uBHap6iTgNuBWt285MB+YDswB7hKR5G76vBW4zfW1y/UdFYemE4+wpGKMMcGieaYyC6hSVb+qNgOLgLkdtpkLPOSWlwKzRURc+yJVPaCq1UCV6y9kn26fs10fuD7/IVpvzBdwhSTzhkTrEMYYMyBFM6mMATYHva5xbSG3UdUWoAHI72Lfztrzgd2uj86OBYCIXC0ilSJSGQgEevG2oCQ/ky8fO4YUKyRpjDFHSLjfiqp6j6pWqGpFYWFhr/qYP2scv5x3TIQjM8aYgS+aSWULMDbodbFrC7mNiKQAOUB9F/t21l4P5Lo+OjuWMcaYKItmUlkJlLlZWWl4A+/LOmyzDLjcLc8DXlJVde3z3eywCUAZsKKzPt0+L7s+cH0+GcX3ZowxJoSU7jfpHVVtEZFrgeeAZGChqq4RkZuBSlVdBtwPPCIiVcBOvCSB224JsBZoAa5R1VaAUH26Q/4UWCQiPwfed30bY4zpR+L9kZ+YKioqtLKyMtZhGGPMgCIi76pqRah1CTdQb4wxJnosqRhjjIkYSyrGGGMixpKKMcaYiEnogXoRCQCf9nL3AqAuguFEisXVMxZXz1hcPROvcUHfYhuvqiHvHk/opNIXIlLZ2eyHWLK4esbi6hmLq2fiNS6IXmx2+csYY0zEWFIxxhgTMZZUeu+eWAfQCYurZyyunrG4eiZe44IoxWZjKsYYYyLGzlSMMcZEjCUVY4wxEWNJpRdEZI6IrBeRKhG5vh+Ot1FEPhKRD0Sk0rUNF5EXRGSD+57n2kVE7nCxfSgixwX1c7nbfoOIXN7Z8bqJZaGI1IrI6qC2iMUiIse791rl9pU+xHWTiGxxn9sHInJh0Lob3DHWi8j5Qe0hf7bucQvLXfti9+iF7mIaKyIvi8haEVkjIt+Ph8+ri7hi+nm5/TJEZIWIrHKx/WdX/Yn3eIzFrn25iJT0NuZexvWgiFQHfWYzXXt//ttPFpH3ReSv8fBZoar21YMvvJL7PmAikAasAsqjfMyNQEGHtl8C17vl64Fb3fKFwDOAACcBy137cMDvvue55bxexHIGcBywOhqx4D035yS3zzPABX2I6ybgJyG2LXc/t3Rggvt5Jnf1swWWAPPd8t3At8OIaRRwnFseBnzijh3Tz6uLuGL6ebltBRjqllOB5e79hewP+A5wt1ueDyzubcy9jOtBYF6I7fvz3/6PgMeAv3b12ffXZ2VnKj03C6hSVb+qNgOLgLkxiGMu8JBbfgj4h6D2h9XzDt4TMUcB5wMvqOpOVd0FvADM6elBVfU1vGffRDwWty5bVd9R71/7w0F99SauzswFFqnqAVWtBqrwfq4hf7buL8azgaUh3mNXMW1T1ffc8h7gY2AMMf68uoirM/3yebl4VFUb3ctU96Vd9Bf8WS4FZrvj9yjmPsTVmX75WYpIMXARcJ973dVn3y+flSWVnhsDbA56XUPX/yEjQYHnReRdEbnatRWp6ja3vB0o6ia+aMYdqVjGuOVIxnitu/ywUNxlpl7ElQ/sVtWW3sblLjUci/cXbtx8Xh3igjj4vNzlnA+AWrxfur4u+jsUg1vf4I4f8f8HHeNS1fbP7Bb3md0mIukd4wrz+L39Wd4OXAe0udddffb98llZUhkYTlPV44ALgGtE5Izgle4vm7iYGx5PsQB/AEqBmcA24L9jEYSIDAX+BPxAVT8LXhfLzytEXHHxealqq6rOBIrx/lqeGos4OuoYl4gcBdyAF98JeJe0ftpf8YjIF4FaVX23v44ZDksqPbcFGBv0uti1RY2qbnHfa4E/4/1H2+FOmXHfa7uJL5pxRyqWLW45IjGq6g73i6ANuBfvc+tNXPV4ly9SOrR3S0RS8X5xP6qqT7jmmH9eoeKKh88rmKruBl4GTu6iv0MxuPU57vhR+38QFNccdylRVfUA8AC9/8x687M8FbhYRDbiXZo6G/gtsf6suht0sa/PDYql4A2uTeDw4NX0KB4vCxgWtPwW3ljIrzhysPeXbvkijhwgXOHahwPVeIODeW55eC9jKuHIAfGIxcLnBysv7ENco4KWf4h33RhgOkcOTPrxBiU7/dkCf+TIwc/vhBGP4F0bv71De0w/ry7iiunn5bYtBHLd8hDgdeCLnfUHXMORg89LehtzL+MaFfSZ3g78Ikb/9s/i8EB9bD+r3vxSSfQvvJkdn+Bd6/1ZlI810f0wVwFr2o+Hdy3078AG4MWgf5gC3Oli+wioCOrrCrxBuCrgm72M53G8SyMH8a6xXhnJWIAKYLXb5/e4qg+9jOsRd9wPgWUc+UvzZ+4Y6wmaZdPZz9b9HFa4eP8IpIcR02l4l7Y+BD5wXxfG+vPqIq6Yfl5uvxnA+y6G1cCNXfUHZLjXVW79xN7G3Mu4XnKf2Wrgfzk8Q6zf/u27fc/icFKJ6WdlZVqMMcZEjI2pGGOMiRhLKsYYYyLGkooxxpiIsaRijDEmYiypGGOMiRhLKsb0kIjkB1Wl3S5HVvbtshqviFSIyB09PN4VrnrthyKyWkTmuvZviMjovrwXYyLNphQb0wcichPQqKq/DmpL0cO1l/rafzHwKl5V4QZXWqVQVatF5BW8qsKVkTiWMZFgZyrGRIB7rsbdIrIc+KWIzBKRt91zLt4SkSluu7OCnntxkyvc+IqI+EXkeyG6HgHsARoBVLXRJZR5eDfLPerOkIa453G86gqPPhdUCuYVEfmt2261iMwKcRxjIsKSijGRUwycoqo/AtYBp6vqscCNwH91ss9UvHLos4D/cDW5gq0CdgDVIvKAiHwJQFWXApXAZeoVOWwBfof3bI/jgYXALUH9ZLrtvuPWGRMVKd1vYowJ0x9VtdUt5wAPiUgZXkmUjsmi3d/UK0Z4QERq8crgHyqBrqqtIjIHrwrubOA2ETleVW/q0M8U4CjgBe8RGSTjla1p97jr7zURyRaRXPUKIxoTUZZUjImcpqDl/w94WVW/7J5Z8kon+xwIWm4lxP9J9QY+VwArROQFvGq4N3XYTIA1qnpyJ8fpOHhqg6kmKuzylzHRkcPhMuHf6G0nIjJagp5vjvesk0/d8h68xwGDVwiwUEROdvulisj0oP0ude2nAQ2q2tDbmIzpip2pGBMdv8S7/PVvwN/60E8q8Gs3dXg/EAC+5dY9CNwtIvvwnjkyD7hDRHLw/m/fjlfZGmC/iLzv+ruiD/EY0yWbUmzMIGdTj01/sstfxhhjIsbOVIwxxkSMnakYY4yJGEsqxhhjIsaSijHGmIixpGKMMSZiLKkYY4yJmP8HLnCrOoiHd3sAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "YgkDE7hzo8r5"
      },
      "source": [
        "## Loss and metrics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MlhsJMm0TW_B",
        "colab": {}
      },
      "source": [
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "67oqVHiT0Eiu",
        "colab": {}
      },
      "source": [
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "  \n",
        "  return tf.reduce_sum(loss_)/tf.reduce_sum(mask)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "phlyxMnm-Tpx",
        "colab": {}
      },
      "source": [
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n",
        "    name='train_accuracy')\n",
        "valid_loss = tf.keras.metrics.Mean(name='valid_loss')\n",
        "valid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n",
        "    name='valid_accuracy')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "aeHumfr7zmMa"
      },
      "source": [
        "## Training and checkpointing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UiysUa--4tOU",
        "colab": {}
      },
      "source": [
        "transformer = Transformer(num_layers, d_model, num_heads, dff,\n",
        "                          input_vocab_size, target_vocab_size, \n",
        "                          pe_input=input_vocab_size, \n",
        "                          pe_target=target_vocab_size,\n",
        "                          rate=dropout_rate)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ZOJUSB1T8GjM",
        "colab": {}
      },
      "source": [
        "def create_masks(inp, tar):\n",
        "  # Encoder padding mask\n",
        "  enc_padding_mask = create_padding_mask(inp)\n",
        "  \n",
        "  # Used in the 2nd attention block in the decoder.\n",
        "  # This padding mask is used to mask the encoder outputs.\n",
        "  dec_padding_mask = create_padding_mask(inp)\n",
        "  \n",
        "  # Used in the 1st attention block in the decoder.\n",
        "  # It is used to pad and mask future tokens in the input received by \n",
        "  # the decoder.\n",
        "  look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
        "  dec_target_padding_mask = create_padding_mask(tar)\n",
        "  combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
        "  \n",
        "  return enc_padding_mask, combined_mask, dec_padding_mask"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Fzuf06YZp66w"
      },
      "source": [
        "Create the checkpoint path and the checkpoint manager. This will be used to save checkpoints every `n` epochs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a78VOJjmfEDR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ckpt_path = \"./checkpoints/w2w/unformated_en_2_formated_fr/cycle6\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oSPCgtpceZdK",
        "colab_type": "text"
      },
      "source": [
        "**real ckpt**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "hNhuYfllndLZ",
        "colab": {}
      },
      "source": [
        "checkpoint_path = ckpt_path + \"/ckpt_real\"\n",
        "\n",
        "ckpt = tf.train.Checkpoint(transformer=transformer,\n",
        "                           optimizer=optimizer)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vcVfqgaZeXCL",
        "colab_type": "text"
      },
      "source": [
        "**virtual ckpt**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ADpNV7qteVGt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "88746606-8c89-44d5-c6f7-ba45aade78cb"
      },
      "source": [
        "checkpoint_path = ckpt_path + \"/ckpt_virtual\"\n",
        "\n",
        "ckpt = tf.train.Checkpoint(transformer=transformer,\n",
        "                           optimizer=optimizer)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=100)"
      ],
      "execution_count": 143,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.iter\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_1\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_2\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.decay\n",
            "WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/guide/checkpoint#loading_mechanics for details.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pLMCL56X7_n3",
        "colab_type": "text"
      },
      "source": [
        "**Restore Ckpt**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6wrdQau7Basv",
        "colab_type": "code",
        "outputId": "6901a68e-8d5d-4950-c96e-5444d60dc4fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# if a checkpoint exists, restore the latest checkpoint.\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "  ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "  print ('Latest checkpoint restored!!')"
      ],
      "execution_count": 144,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Latest checkpoint restored!!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iJwmp9OE29oj",
        "colab": {}
      },
      "source": [
        "# The @tf.function trace-compiles train_step into a TF graph for faster\n",
        "# execution. The function specializes to the precise shape of the argument\n",
        "# tensors. To avoid re-tracing due to the variable sequence lengths or variable\n",
        "# batch sizes (the last batch is smaller), use input_signature to specify\n",
        "# more generic shapes.\n",
        "\n",
        "train_step_signature = [\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int32),\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int32),\n",
        "]\n",
        "\n",
        "@tf.function(input_signature=train_step_signature)\n",
        "def train_step(inp, tar):\n",
        "  tar_inp = tar[:, :-1]\n",
        "  tar_real = tar[:, 1:]\n",
        "  enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
        "  with tf.GradientTape() as tape:\n",
        "    predictions, _ = transformer(inp, tar_inp, \n",
        "                                 True, \n",
        "                                 enc_padding_mask, \n",
        "                                 combined_mask, \n",
        "                                 dec_padding_mask)\n",
        "    loss = loss_function(tar_real, predictions)\n",
        "  gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
        "  optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
        "  train_loss(loss)\n",
        "  train_accuracy(tar_real, predictions)\n",
        "\n",
        "@tf.function(input_signature=train_step_signature)\n",
        "def valid_step(inp, tar):\n",
        "  tar_inp = tar[:, :-1]\n",
        "  tar_real = tar[:, 1:]\n",
        "  enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
        "  predictions, _ = transformer(inp, tar_inp, \n",
        "                                True, \n",
        "                                enc_padding_mask, \n",
        "                                combined_mask, \n",
        "                                dec_padding_mask)\n",
        "  loss = loss_function(tar_real, predictions)\n",
        "  valid_loss(loss)\n",
        "  valid_accuracy(tar_real, predictions)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZnvwR9tZYz7A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "EPOCHS = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bbvmaKNiznHZ",
        "outputId": "864e0811-d198-4561-8b4a-563f4724d5f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "3719dc5d1d1e47f2a84b75194970f11b",
            "f99de86c50da449ebaa87ff8e172e74a",
            "1463cbfaf0544e418e197e35c565f612",
            "450677d6c74b45ee83902dbf96f1a257",
            "4ef048fad07a4772940f209a95c04eec",
            "6f4bd0d5f3394a479043a45cebc12a0a",
            "117ea04694884baa92c55e70abc04310",
            "14b4dd74a5744adda8dd07bc3a8cb2b8"
          ]
        }
      },
      "source": [
        "D={'T_loss':[], 'V_loss':[], 'T_Acc':[], 'V_Acc':[], 'E':[]}\n",
        "for epoch in tqdm(range(EPOCHS)):\n",
        "  start = time.time()\n",
        "  \n",
        "  train_loss.reset_states()\n",
        "  train_accuracy.reset_states()\n",
        "  \n",
        "  valid_loss.reset_states()\n",
        "  valid_accuracy.reset_states()\n",
        "  \n",
        "\n",
        "  # inp -> portuguese, tar -> english\n",
        "  for (batch, (inp, tar)) in enumerate(dataset_train):\n",
        "    train_step(inp, tar)\n",
        "    \n",
        "    if batch % 100 == 0:\n",
        "      print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
        "          epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n",
        "\n",
        "  for (batch, (v_inp, v_tar)) in enumerate(dataset_valid):\n",
        "    valid_step(v_inp, v_tar)\n",
        "\n",
        "  print ('Vaildation Epoch {} Val_Loss {:.4f}, Val_Acc {:.4f}'.format(\n",
        "          epoch + 1, valid_loss.result(), valid_accuracy.result()))\n",
        "  \n",
        "  D['T_loss'].append(train_loss.result().numpy())\n",
        "  D['V_loss'].append(valid_loss.result().numpy())\n",
        "  D['T_Acc'].append(train_accuracy.result().numpy())\n",
        "  D['V_Acc'].append(valid_accuracy.result().numpy())\n",
        "  D['E'].append(epoch + 1)\n",
        "\n",
        "\n",
        "  if (epoch + 1) % 1 == 0:\n",
        "    ckpt_save_path = ckpt_manager.save()\n",
        "    print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,\n",
        "                                                         ckpt_save_path))\n",
        "    \n",
        "  print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, \n",
        "                                                train_loss.result(), \n",
        "                                                train_accuracy.result()))\n",
        "\n",
        "  print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
      ],
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3719dc5d1d1e47f2a84b75194970f11b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 Batch 0 Loss 3.8244 Accuracy 0.0808\n",
            "Epoch 1 Batch 100 Loss 3.3229 Accuracy 0.0890\n",
            "Epoch 1 Batch 200 Loss 3.2136 Accuracy 0.0898\n",
            "Vaildation Epoch 1 Val_Loss 3.3906, Val_Acc 0.0874\n",
            "Saving checkpoint for epoch 1 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-11\n",
            "Epoch 1 Loss 3.1501 Accuracy 0.0910\n",
            "Time taken for 1 epoch: 57.80599784851074 secs\n",
            "\n",
            "Epoch 2 Batch 0 Loss 2.9520 Accuracy 0.1010\n",
            "Epoch 2 Batch 100 Loss 2.9254 Accuracy 0.0967\n",
            "Epoch 2 Batch 200 Loss 2.8875 Accuracy 0.0955\n",
            "Vaildation Epoch 2 Val_Loss 3.3846, Val_Acc 0.0875\n",
            "Saving checkpoint for epoch 2 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-12\n",
            "Epoch 2 Loss 2.8545 Accuracy 0.0959\n",
            "Time taken for 1 epoch: 46.303186893463135 secs\n",
            "\n",
            "Epoch 3 Batch 0 Loss 2.7239 Accuracy 0.1037\n",
            "Epoch 3 Batch 100 Loss 2.7203 Accuracy 0.0994\n",
            "Epoch 3 Batch 200 Loss 2.6858 Accuracy 0.0982\n",
            "Vaildation Epoch 3 Val_Loss 3.3961, Val_Acc 0.0882\n",
            "Saving checkpoint for epoch 3 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-13\n",
            "Epoch 3 Loss 2.6589 Accuracy 0.0985\n",
            "Time taken for 1 epoch: 46.30555033683777 secs\n",
            "\n",
            "Epoch 4 Batch 0 Loss 2.5123 Accuracy 0.1026\n",
            "Epoch 4 Batch 100 Loss 2.5382 Accuracy 0.1012\n",
            "Epoch 4 Batch 200 Loss 2.5115 Accuracy 0.0998\n",
            "Vaildation Epoch 4 Val_Loss 3.4377, Val_Acc 0.0874\n",
            "Saving checkpoint for epoch 4 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-14\n",
            "Epoch 4 Loss 2.4911 Accuracy 0.1002\n",
            "Time taken for 1 epoch: 46.423933267593384 secs\n",
            "\n",
            "Epoch 5 Batch 0 Loss 2.3515 Accuracy 0.1069\n",
            "Epoch 5 Batch 100 Loss 2.3850 Accuracy 0.1032\n",
            "Epoch 5 Batch 200 Loss 2.3595 Accuracy 0.1019\n",
            "Vaildation Epoch 5 Val_Loss 3.4576, Val_Acc 0.0877\n",
            "Saving checkpoint for epoch 5 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-15\n",
            "Epoch 5 Loss 2.3460 Accuracy 0.1021\n",
            "Time taken for 1 epoch: 46.424341440200806 secs\n",
            "\n",
            "Epoch 6 Batch 0 Loss 2.2459 Accuracy 0.1076\n",
            "Epoch 6 Batch 100 Loss 2.2583 Accuracy 0.1052\n",
            "Epoch 6 Batch 200 Loss 2.2368 Accuracy 0.1039\n",
            "Vaildation Epoch 6 Val_Loss 3.4749, Val_Acc 0.0881\n",
            "Saving checkpoint for epoch 6 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-16\n",
            "Epoch 6 Loss 2.2241 Accuracy 0.1041\n",
            "Time taken for 1 epoch: 46.342437505722046 secs\n",
            "\n",
            "Epoch 7 Batch 0 Loss 2.1276 Accuracy 0.1067\n",
            "Epoch 7 Batch 100 Loss 2.1478 Accuracy 0.1067\n",
            "Epoch 7 Batch 200 Loss 2.1323 Accuracy 0.1053\n",
            "Vaildation Epoch 7 Val_Loss 3.4983, Val_Acc 0.0878\n",
            "Saving checkpoint for epoch 7 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-17\n",
            "Epoch 7 Loss 2.1222 Accuracy 0.1055\n",
            "Time taken for 1 epoch: 46.32657170295715 secs\n",
            "\n",
            "Epoch 8 Batch 0 Loss 2.0318 Accuracy 0.1091\n",
            "Epoch 8 Batch 100 Loss 2.0520 Accuracy 0.1086\n",
            "Epoch 8 Batch 200 Loss 2.0419 Accuracy 0.1069\n",
            "Vaildation Epoch 8 Val_Loss 3.5385, Val_Acc 0.0867\n",
            "Saving checkpoint for epoch 8 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-18\n",
            "Epoch 8 Loss 2.0346 Accuracy 0.1071\n",
            "Time taken for 1 epoch: 46.45223069190979 secs\n",
            "\n",
            "Epoch 9 Batch 0 Loss 1.9951 Accuracy 0.1130\n",
            "Epoch 9 Batch 100 Loss 1.9744 Accuracy 0.1097\n",
            "Epoch 9 Batch 200 Loss 1.9598 Accuracy 0.1082\n",
            "Vaildation Epoch 9 Val_Loss 3.5646, Val_Acc 0.0861\n",
            "Saving checkpoint for epoch 9 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-19\n",
            "Epoch 9 Loss 1.9520 Accuracy 0.1086\n",
            "Time taken for 1 epoch: 46.226736068725586 secs\n",
            "\n",
            "Epoch 10 Batch 0 Loss 1.8543 Accuracy 0.1123\n",
            "Epoch 10 Batch 100 Loss 1.8945 Accuracy 0.1116\n",
            "Epoch 10 Batch 200 Loss 1.8839 Accuracy 0.1101\n",
            "Vaildation Epoch 10 Val_Loss 3.6022, Val_Acc 0.0854\n",
            "Saving checkpoint for epoch 10 at ./checkpoints/w2w/unformated_en_2_formated_fr/cycle6/ckpt_virtual/ckpt-20\n",
            "Epoch 10 Loss 1.8803 Accuracy 0.1102\n",
            "Time taken for 1 epoch: 46.357967138290405 secs\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcAPwjbrYo9l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "log_dir = os.path.join(ckpt_path, \"log_real.npy\")\n",
        "os.makedirs(os.path.dirname(log_dir), exist_ok=True)\n",
        "np.save(log_dir, D)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "QfcsSWswSdGV"
      },
      "source": [
        "## Evaluate"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "y6APsFrgImLW"
      },
      "source": [
        "The following steps are used for evaluation:\n",
        "\n",
        "* Encode the input sentence using the Portuguese tokenizer (`tokenizer_pt`). Moreover, add the start and end token so the input is equivalent to what the model is trained with. This is the encoder input.\n",
        "* The decoder input is the `start token == tokenizer_en.vocab_size`.\n",
        "* Calculate the padding masks and the look ahead masks.\n",
        "* The `decoder` then outputs the predictions by looking at the `encoder output` and its own output (self-attention).\n",
        "* Select the last word and calculate the argmax of that.\n",
        "* Concatentate the predicted word to the decoder input as pass it to the decoder.\n",
        "* In this approach, the decoder predicts the next word based on the previous words it predicted.\n",
        "\n",
        "Note: The model used here has less capacity to keep the example relatively faster so the predictions maybe less right. To reproduce the results in the paper, use the entire dataset and base transformer model or transformer XL, by changing the hyperparameters above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "5buvMlnvyrFm",
        "colab": {}
      },
      "source": [
        "def evaluate(encoder_input):\n",
        "\n",
        "  batch_size = encoder_input.shape[0]\n",
        "  output = np.array([decoder_v2id['<SOS>']]*batch_size).reshape(-1,1)\n",
        "  MAX_LENGTH = encoder_input.shape[-1]\n",
        "\n",
        "  for i in range(MAX_LENGTH):\n",
        "    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n",
        "        encoder_input, output)\n",
        "  \n",
        "    # predictions.shape == (batch_size, seq_len, vocab_size)\n",
        "    predictions, attention_weights = transformer(encoder_input, \n",
        "                                                 output,\n",
        "                                                 False,\n",
        "                                                 enc_padding_mask,\n",
        "                                                 combined_mask,\n",
        "                                                 dec_padding_mask)\n",
        "    \n",
        "    # select the last word from the seq_len dimension\n",
        "    predictions = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\n",
        "    predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
        "    \n",
        "    # return the result if the predicted_id is equal to the end token\n",
        "    # if predicted_id == tokenizer_en.vocab_size+1:\n",
        "    #   return output, attention_weights\n",
        "    \n",
        "    # concatentate the predicted_id to the output which is given to the decoder\n",
        "    # as its input.\n",
        "    output = tf.concat([output, predicted_id], axis=1)\n",
        "\n",
        "  return output, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "onj-LdrNvtIn",
        "colab_type": "text"
      },
      "source": [
        "### BLEU Score Check"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "USjPs21xxiAy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "enc_gen = get_gen(data=aligned_en, batch_size=32, valid_test=True)\n",
        "dec_gen = get_gen(data=aligned_fr, batch_size=32, valid_test=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RDBUoLJ-0zod",
        "colab_type": "code",
        "outputId": "73c19f73-e55d-42fc-f67a-d4b344f2f9e0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 899,
          "referenced_widgets": [
            "8a88e732e81b447eaa7db435501e468c",
            "acf00b053c1d4ea7acf3963728bf0471",
            "4d9a3801f9fd48859cbecb63c07f948d",
            "ffef32d4364d4b9da9fc03f739f0e3c6",
            "d62751787d8a4547859e76249cb1070a",
            "de37452ce97645acb3ca9423781cdd0b",
            "179b8510943f49de828a367820fdb184",
            "b4e1dcc68e2043c1a8f91bb69e7f3474"
          ]
        }
      },
      "source": [
        "Sum = 0\n",
        "Total_n = 0\n",
        "for enc_data_words, dec_data_words in tqdm(zip(enc_gen, dec_gen)):\n",
        "    out_words = get_out(enc_data_words)\n",
        "    Sum += sum([sacrebleu.sentence_bleu(out_words[i], dec_data_words[i], smooth_method='exp').score for i in range(len(out_words))])\n",
        "    Total_n += len(out_words)\n",
        "    print(Sum/Total_n, \"Out of {} samples\".format(Total_n))"
      ],
      "execution_count": 151,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8a88e732e81b447eaa7db435501e468c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 38.88it/s]\n",
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 38.57it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10.860525496561465 Out of 32 samples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 38.39it/s]\n",
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 38.16it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "8.99795817427104 Out of 64 samples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 38.34it/s]\n",
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 38.08it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "8.992966103423173 Out of 96 samples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 37.88it/s]\n",
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 37.42it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "8.770726614895333 Out of 128 samples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 37.69it/s]\n",
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 38.90it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "8.747746702002177 Out of 160 samples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 37.08it/s]\n",
            "\n",
            "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 32/32 [00:00<00:00, 34.81it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "8.568686825260636 Out of 192 samples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/28 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 28/28 [00:00<00:00, 33.63it/s]\n",
            "\n",
            "  0%|          | 0/28 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 28/28 [00:00<00:00, 30.85it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "8.726709854889316 Out of 220 samples\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I_zCe2v8lehs",
        "colab_type": "text"
      },
      "source": [
        "## Prediction and virtual file creation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_36nPSVjoH5a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "new_unaligned_en = '/content/drive/My Drive/ift6759/ift6759-t6-p2/data/cycle5/unaligned_unformated_en'\n",
        "new_unaligned_fr = '/content/drive/My Drive/ift6759/ift6759-t6-p2/data/cycle5/unaligned_formated_fr'\n",
        "os.makedirs(os.path.dirname(new_unaligned_en), exist_ok=True)\n",
        "os.makedirs(os.path.dirname(new_unaligned_fr), exist_ok=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rcd6IQVWmMSe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "enc_gen = get_gen(data=original_unaligned_en, batch_size=256, valid_test=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6yjAfaBGlj7r",
        "colab_type": "code",
        "outputId": "d80acf5a-90de-4039-fece-685a0bf7cd1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "e6c7cbf209c24264a30465c2c272fd5a",
            "7adac9bb16bc4acf8796aefe21ebd508",
            "026da294751543e5946394f90a107d12",
            "4b30a7016fd24f17973fd5b2c4f80636",
            "ee4eb2bae55f4c3fb6f49bc801697e42",
            "ead005a0cb024ce5bbde87e4339b5eb9",
            "3a910fe9cd374eb6aa39cd61d7309ca2",
            "1bc9ca6a36cb4e28893932dc9a0cb737"
          ]
        }
      },
      "source": [
        "start_point = 200\n",
        "required = start_point + 200\n",
        "for i, enc_data_words in tqdm(enumerate(enc_gen), total=required):\n",
        "    if required is None or (i<required and i>=start_point):\n",
        "        try:\n",
        "            out_words = get_out(enc_data_words)\n",
        "        except Exception:\n",
        "            print(\"OOM\")\n",
        "            required += 1\n",
        "            continue\n",
        "        write_file_append(data=enc_data_words, output=new_unaligned_en)\n",
        "        write_file_append(data=out_words, output=new_unaligned_fr)"
      ],
      "execution_count": 462,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e6c7cbf209c24264a30465c2c272fd5a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=400), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[1;30;43mStreaming output truncated to the last 5000 lines.\u001b[0m\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 199.22it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 229.74it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:30,  1.06s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 45%|████▌     | 116/256 [00:01<01:44,  1.34it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 196.52it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 228.74it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:14,  1.89it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 206.70it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 231.51it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:27,  1.05s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 207.06it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.31it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:14,  1.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 53%|█████▎    | 136/256 [00:01<00:44,  2.71it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 199.21it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 231.55it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:30,  1.06s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 48%|████▊     | 123/256 [00:01<01:38,  1.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 198.97it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 231.11it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:29,  1.06s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 205.54it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 232.94it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:31,  1.07s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 53%|█████▎    | 136/256 [00:01<01:29,  1.34it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 195.08it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.69it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:33,  1.07s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 44%|████▍     | 112/256 [00:01<01:48,  1.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 194.20it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 228.50it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  2%|▏         | 4/256 [00:01<01:07,  3.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 202.24it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.72it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  4%|▍         | 10/256 [00:01<00:25,  9.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 48%|████▊     | 123/256 [00:01<00:09, 13.49it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 201.61it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 229.67it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:29,  2.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 39%|███▉      | 100/256 [00:01<00:38,  4.04it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 195.43it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 232.60it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:31,  1.07s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 66%|██████▌   | 169/256 [00:01<01:04,  1.34it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 199.49it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.44it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  3%|▎         | 7/256 [00:01<00:37,  6.67it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 206.53it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.10it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:29,  2.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 32%|███▏      | 83/256 [00:01<00:42,  4.04it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 191.90it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.74it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:25,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 208.92it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.29it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:12,  1.91it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 44%|████▍     | 112/256 [00:01<00:52,  2.73it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 201.72it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.48it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:27,  2.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 43%|████▎     | 111/256 [00:01<00:35,  4.11it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 203.00it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 237.41it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:26,  1.05s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 207.78it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 237.68it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:26,  1.05s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 205.39it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.19it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:27,  1.05s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 38%|███▊      | 98/256 [00:01<01:56,  1.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 198.40it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.04it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:12,  1.92it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 209.24it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 240.90it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 39%|███▉      | 101/256 [00:01<01:52,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 197.63it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.43it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:11,  1.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 34%|███▍      | 87/256 [00:01<01:01,  2.76it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 196.95it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.86it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 52%|█████▏    | 132/256 [00:01<01:29,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 202.13it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 232.20it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:24,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 29%|██▊       | 73/256 [00:01<02:12,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 197.72it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.61it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:11,  1.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 208.46it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 237.75it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  2%|▏         | 5/256 [00:01<00:51,  4.84it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 45%|████▌     | 116/256 [00:01<00:20,  6.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 206.64it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 238.43it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  2%|▏         | 4/256 [00:01<01:05,  3.84it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 30%|███       | 77/256 [00:01<00:32,  5.47it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 200.58it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.03it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:22,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 42%|████▏     | 108/256 [00:01<01:46,  1.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 198.64it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.69it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 43%|████▎     | 110/256 [00:01<01:45,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 198.46it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.42it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:27,  2.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 46%|████▋     | 119/256 [00:01<00:33,  4.12it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 197.57it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.58it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:27,  1.05s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 38%|███▊      | 98/256 [00:01<01:55,  1.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 198.58it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 237.70it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:12,  1.91it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 35%|███▍      | 89/256 [00:01<01:01,  2.73it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 196.10it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.87it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:27,  2.91it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 50%|████▉     | 127/256 [00:01<00:31,  4.15it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 199.77it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.50it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:11,  1.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 55%|█████▌    | 141/256 [00:01<00:41,  2.76it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 200.80it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.91it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:24,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 209.05it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 227.25it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:27,  2.89it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 43%|████▎     | 109/256 [00:01<00:35,  4.13it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 194.96it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 238.68it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:26,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 52%|█████▏    | 132/256 [00:01<01:30,  1.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 200.56it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 231.53it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:31,  1.06s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 47%|████▋     | 120/256 [00:01<01:41,  1.34it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 190.43it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.66it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 44%|████▍     | 113/256 [00:01<01:43,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 205.48it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 237.01it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:28,  2.87it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 35%|███▍      | 89/256 [00:01<00:40,  4.10it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 196.39it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.83it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  2%|▏         | 4/256 [00:01<01:05,  3.86it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 205.95it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.67it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:22,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 209.02it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.93it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:24,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 50%|█████     | 129/256 [00:01<01:32,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 203.90it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.48it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:10,  1.94it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 37%|███▋      | 94/256 [00:01<00:58,  2.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 195.22it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.73it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:22,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 43%|████▎     | 110/256 [00:01<01:45,  1.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 207.18it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.54it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:10,  1.95it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 58%|█████▊    | 148/256 [00:01<00:38,  2.78it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 205.50it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.78it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:22,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 42%|████▏     | 107/256 [00:01<01:47,  1.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 202.45it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.28it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:11,  1.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 40%|███▉      | 102/256 [00:01<00:55,  2.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 198.46it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.99it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:27,  2.89it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 54%|█████▍    | 139/256 [00:01<00:28,  4.12it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 203.02it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.93it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 51%|█████     | 130/256 [00:01<01:31,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 203.02it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.33it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:12,  1.92it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 49%|████▉     | 125/256 [00:01<00:47,  2.74it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 201.32it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 231.41it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:27,  2.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 44%|████▍     | 113/256 [00:01<00:34,  4.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 192.92it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.08it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  2%|▏         | 4/256 [00:01<01:05,  3.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 46%|████▋     | 119/256 [00:01<00:24,  5.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 201.10it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.70it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:11,  1.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 207.78it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 239.03it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:12,  1.92it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 41%|████      | 105/256 [00:01<00:55,  2.74it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 200.87it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.33it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:22,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 42%|████▏     | 108/256 [00:01<01:46,  1.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 200.47it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 227.91it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:22,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 44%|████▍     | 112/256 [00:01<01:43,  1.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 202.09it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.75it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:11,  1.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 55%|█████▍    | 140/256 [00:01<00:42,  2.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 201.74it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 238.74it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:21,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 52%|█████▏    | 134/256 [00:01<01:27,  1.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 203.55it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 237.36it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 51%|█████     | 130/256 [00:01<01:31,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 201.84it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 228.89it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 209.76it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.27it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:20,  1.02s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 45%|████▌     | 116/256 [00:01<01:40,  1.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 204.34it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 233.89it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:18,  1.01s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 213.08it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 239.30it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  2%|▏         | 4/256 [00:01<01:04,  3.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 45%|████▍     | 115/256 [00:01<00:25,  5.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 199.39it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.19it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:25,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 209.31it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 238.94it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:10,  1.94it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 31%|███       | 79/256 [00:01<01:03,  2.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 193.98it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 232.62it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:26,  2.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 210.17it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.68it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 210.76it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 238.47it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:24,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 41%|████      | 104/256 [00:01<01:50,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 193.10it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.06it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 207.01it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 237.55it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:13,  1.91it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 46%|████▋     | 119/256 [00:01<00:50,  2.72it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 201.67it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.71it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:24,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 46%|████▌     | 118/256 [00:01<01:40,  1.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 200.51it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.15it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:25,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 207.33it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 229.86it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:24,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 208.54it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.91it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:23,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 210.17it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 230.42it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:11,  1.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 44%|████▍     | 113/256 [00:01<00:51,  2.76it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 206.77it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.07it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:25,  1.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 44%|████▍     | 113/256 [00:01<01:44,  1.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 201.28it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.63it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:10,  1.94it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 46%|████▋     | 119/256 [00:01<00:49,  2.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 205.37it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.74it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:26,  2.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 50%|████▉     | 127/256 [00:01<00:30,  4.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 206.13it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.34it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 1/256 [00:01<04:21,  1.03s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 52%|█████▏    | 132/256 [00:01<01:29,  1.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 203.24it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 231.23it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 3/256 [00:01<01:27,  2.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 209.51it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 234.55it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  2%|▏         | 4/256 [00:01<01:05,  3.87it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 39%|███▉      | 101/256 [00:01<00:28,  5.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 198.98it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 236.19it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  1%|          | 2/256 [00:01<02:11,  1.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " 55%|█████▍    | 140/256 [00:01<00:42,  2.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 203.15it/s]\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "  0%|          | 0/256 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "100%|██████████| 256/256 [00:01<00:00, 235.62it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nb67W9VFEWrq",
        "colab_type": "text"
      },
      "source": [
        "## Plots"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "CN-BV43FMBej",
        "colab": {}
      },
      "source": [
        "def plot_attention_weights(attention, sentence, result, layer):\n",
        "  fig = plt.figure(figsize=(16, 8))\n",
        "  \n",
        "  sentence = tokenizer_pt.encode(sentence)\n",
        "  \n",
        "  attention = tf.squeeze(attention[layer], axis=0)\n",
        "  \n",
        "  for head in range(attention.shape[0]):\n",
        "    ax = fig.add_subplot(2, 4, head+1)\n",
        "    \n",
        "    # plot the attention weights\n",
        "    ax.matshow(attention[head][:-1, :], cmap='viridis')\n",
        "\n",
        "    fontdict = {'fontsize': 10}\n",
        "    \n",
        "    ax.set_xticks(range(len(sentence)+2))\n",
        "    ax.set_yticks(range(len(result)))\n",
        "    \n",
        "    ax.set_ylim(len(result)-1.5, -0.5)\n",
        "        \n",
        "    ax.set_xticklabels(\n",
        "        ['<start>']+[tokenizer_pt.decode([i]) for i in sentence]+['<end>'], \n",
        "        fontdict=fontdict, rotation=90)\n",
        "    \n",
        "    ax.set_yticklabels([tokenizer_en.decode([i]) for i in result \n",
        "                        if i < tokenizer_en.vocab_size], \n",
        "                       fontdict=fontdict)\n",
        "    \n",
        "    ax.set_xlabel('Head {}'.format(head+1))\n",
        "  \n",
        "  plt.tight_layout()\n",
        "  plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lU2_yG_vBGza",
        "colab": {}
      },
      "source": [
        "def translate(sentence, plot=''):\n",
        "  result, attention_weights = evaluate(sentence)\n",
        "  \n",
        "  predicted_sentence = tokenizer_en.decode([i for i in result \n",
        "                                            if i < tokenizer_en.vocab_size])  \n",
        "\n",
        "  print('Input: {}'.format(sentence))\n",
        "  print('Predicted translation: {}'.format(predicted_sentence))\n",
        "  \n",
        "  if plot:\n",
        "    plot_attention_weights(attention_weights, sentence, result, plot)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YsxrAlvFG8SZ",
        "colab": {}
      },
      "source": [
        "translate(\"este é um problema que temos que resolver.\")\n",
        "print (\"Real translation: this is a problem we have to solve .\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7EH5y_aqI4t1",
        "colab": {}
      },
      "source": [
        "translate(\"os meus vizinhos ouviram sobre esta ideia.\")\n",
        "print (\"Real translation: and my neighboring homes heard about this idea .\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "J-hVCTSUMlkb",
        "colab": {}
      },
      "source": [
        "translate(\"vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.\")\n",
        "print (\"Real translation: so i 'll just share with you some stories very quickly of some magical things that have happened .\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "_1MxkSZvz0jX"
      },
      "source": [
        "You can pass different layers and attention blocks of the decoder to the `plot` parameter."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "t-kFyiOLH0xg",
        "colab": {}
      },
      "source": [
        "translate(\"este é o primeiro livro que eu fiz.\", plot='decoder_layer4_block2')\n",
        "print (\"Real translation: this is the first book i've ever done.\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RqQ1fIsLwkGE"
      },
      "source": [
        "## Summary\n",
        "\n",
        "In this tutorial, you learned about positional encoding, multi-head attention, the importance of masking and how to create a transformer.\n",
        "\n",
        "Try using a different dataset to train the transformer. You can also create the base transformer or transformer XL by changing the hyperparameters above. You can also use the layers defined here to create [BERT](https://arxiv.org/abs/1810.04805) and train state of the art models. Futhermore, you can implement beam search to get better predictions."
      ]
    }
  ]
}