visualize_llm_size.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMpzWC/7bH2+sZncPlMtE9B",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/biwa7636/blog/blob/main/visualize_llm_size.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "metadata": {
        "id": "OE8dDTzUWZ8w"
      },
      "outputs": [],
      "source": [
        "%pip install -q transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import plotly.graph_objects as go\n",
        "from transformers import AutoModelForCausalLM, BertModel\n",
        "\n",
        "ckpt = 'bert-base-uncased'\n",
        "model = BertModel.from_pretrained(ckpt)"
      ],
      "metadata": {
        "id": "Nw59V5jbWl2-"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "labels = []\n",
        "parents = []\n",
        "values = []\n",
        "texts = []\n",
        "\n",
        "def traverse(module, parent=''):\n",
        "\n",
        "    parents.append(parent)\n",
        "    texts.append(module.__repr__().replace(\"\\n\", \"<br>\"))\n",
        "\n",
        "    cur_num = sum(p.numel() for p in module.parameters())\n",
        "    values.append(cur_num)\n",
        "\n",
        "    cur_label = parent + '/' + module._get_name()\n",
        "    # confirm cur_label is unique, otherwise, rename\n",
        "    c = 1\n",
        "    while 1:\n",
        "        if cur_label not in labels:\n",
        "            break\n",
        "        elif cur_label + str(c) not in labels:\n",
        "            cur_label += str(c)\n",
        "            break\n",
        "        else:\n",
        "            c += 1\n",
        "    labels.append(cur_label)\n",
        "\n",
        "    for name, child in module.named_children():\n",
        "        traverse(child, parent=cur_label)\n",
        "\n",
        "\n",
        "traverse(model, parent='')"
      ],
      "metadata": {
        "id": "oBrRUr04W42T"
      },
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig = go.Figure(go.Treemap(\n",
        "    labels = labels,\n",
        "    parents = parents,\n",
        "    values = values,\n",
        "    text = texts,\n",
        "    hoverlabel=dict(bgcolor='white')\n",
        "))\n",
        "\n",
        "fig.update_traces(root_color='lightgrey')\n",
        "fig.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "Pz9E-tmkbt16",
        "outputId": "57b2a779-8079-49e7-8c29-2caf0ee1f819"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"d3bb13b7-4a14-422a-8df2-5520ccce4191\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d3bb13b7-4a14-422a-8df2-5520ccce4191\")) {                    Plotly.newPlot(                        \"d3bb13b7-4a14-422a-8df2-5520ccce4191\",                        [{\"hoverlabel\":{\"bgcolor\":\"white\"},\"labels\":[\"/BertModel\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings/Embedding\",\"/BertModel/BertEmbeddings/Embedding1\",\"/BertModel/BertEmbeddings/Embedding2\",\"/BertModel/BertEmbeddings/LayerNorm\",\"/BertModel/BertEmbeddings/Dropout\",\"/BertModel/BertEncoder\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer3\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer4\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer5\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer6\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer7\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer8\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer9\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer10\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer11\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput/Dropout\",\"/BertModel/BertPooler\",\"/BertModel/BertPooler/Linear\",\"/BertModel/BertPooler/Tanh\"],\"parents\":[\"\",\"/BertModel\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings\",\"/BertModel\",\"/BertModel/BertEncoder\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer3\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer3\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer4\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer4\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer5\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer5\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer6\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer6\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer7\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer7\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer8\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer8\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer9\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer9\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer10\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer10\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer11\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer11\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput\",\"/BertModel\",\"/BertModel/BertPooler\",\"/BertModel/BertPooler\"],\"text\":[\"BertModel(<br>  (embeddings): BertEmbeddings(<br>    (word_embeddings): Embedding(30522, 768, padding_idx=0)<br>    (position_embeddings): Embedding(512, 768)<br>    (token_type_embeddings): Embedding(2, 768)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (encoder): BertEncoder(<br>    (layer): ModuleList(<br>      (0-11): 12 x BertLayer(<br>        (attention): BertAttention(<br>          (self): BertSelfAttention(<br>            (query): Linear(in_features=768, out_features=768, bias=True)<br>            (key): Linear(in_features=768, out_features=768, bias=True)<br>            (value): Linear(in_features=768, out_features=768, bias=True)<br>            (dropout): Dropout(p=0.1, inplace=False)<br>          )<br>          (output): BertSelfOutput(<br>            (dense): Linear(in_features=768, out_features=768, bias=True)<br>            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>            (dropout): Dropout(p=0.1, inplace=False)<br>          )<br>        )<br>        (intermediate): BertIntermediate(<br>          (dense): Linear(in_features=768, out_features=3072, bias=True)<br>          (intermediate_act_fn): GELUActivation()<br>        )<br>        (output): BertOutput(<br>          (dense): Linear(in_features=3072, out_features=768, bias=True)<br>          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>          (dropout): Dropout(p=0.1, inplace=False)<br>        )<br>      )<br>    )<br>  )<br>  (pooler): BertPooler(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (activation): Tanh()<br>  )<br>)\",\"BertEmbeddings(<br>  (word_embeddings): Embedding(30522, 768, padding_idx=0)<br>  (position_embeddings): Embedding(512, 768)<br>  (token_type_embeddings): Embedding(2, 768)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Embedding(30522, 768, padding_idx=0)\",\"Embedding(512, 768)\",\"Embedding(2, 768)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertEncoder(<br>  (layer): ModuleList(<br>    (0-11): 12 x BertLayer(<br>      (attention): BertAttention(<br>        (self): BertSelfAttention(<br>          (query): Linear(in_features=768, out_features=768, bias=True)<br>          (key): Linear(in_features=768, out_features=768, bias=True)<br>          (value): Linear(in_features=768, out_features=768, bias=True)<br>          (dropout): Dropout(p=0.1, inplace=False)<br>        )<br>        (output): BertSelfOutput(<br>          (dense): Linear(in_features=768, out_features=768, bias=True)<br>          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>          (dropout): Dropout(p=0.1, inplace=False)<br>        )<br>      )<br>      (intermediate): BertIntermediate(<br>        (dense): Linear(in_features=768, out_features=3072, bias=True)<br>        (intermediate_act_fn): GELUActivation()<br>      )<br>      (output): BertOutput(<br>        (dense): Linear(in_features=3072, out_features=768, bias=True)<br>        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>        (dropout): Dropout(p=0.1, inplace=False)<br>      )<br>    )<br>  )<br>)\",\"ModuleList(<br>  (0-11): 12 x BertLayer(<br>    (attention): BertAttention(<br>      (self): BertSelfAttention(<br>        (query): Linear(in_features=768, out_features=768, bias=True)<br>        (key): Linear(in_features=768, out_features=768, bias=True)<br>        (value): Linear(in_features=768, out_features=768, bias=True)<br>        (dropout): Dropout(p=0.1, inplace=False)<br>      )<br>      (output): BertSelfOutput(<br>        (dense): Linear(in_features=768, out_features=768, bias=True)<br>        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>        (dropout): Dropout(p=0.1, inplace=False)<br>      )<br>    )<br>    (intermediate): BertIntermediate(<br>      (dense): Linear(in_features=768, out_features=3072, bias=True)<br>      (intermediate_act_fn): GELUActivation()<br>    )<br>    (output): BertOutput(<br>      (dense): Linear(in_features=3072, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertLayer(<br>  (attention): BertAttention(<br>    (self): BertSelfAttention(<br>      (query): Linear(in_features=768, out_features=768, bias=True)<br>      (key): Linear(in_features=768, out_features=768, bias=True)<br>      (value): Linear(in_features=768, out_features=768, bias=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>    (output): BertSelfOutput(<br>      (dense): Linear(in_features=768, out_features=768, bias=True)<br>      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>      (dropout): Dropout(p=0.1, inplace=False)<br>    )<br>  )<br>  (intermediate): BertIntermediate(<br>    (dense): Linear(in_features=768, out_features=3072, bias=True)<br>    (intermediate_act_fn): GELUActivation()<br>  )<br>  (output): BertOutput(<br>    (dense): Linear(in_features=3072, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertAttention(<br>  (self): BertSelfAttention(<br>    (query): Linear(in_features=768, out_features=768, bias=True)<br>    (key): Linear(in_features=768, out_features=768, bias=True)<br>    (value): Linear(in_features=768, out_features=768, bias=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>  (output): BertSelfOutput(<br>    (dense): Linear(in_features=768, out_features=768, bias=True)<br>    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>    (dropout): Dropout(p=0.1, inplace=False)<br>  )<br>)\",\"BertSelfAttention(<br>  (query): Linear(in_features=768, out_features=768, bias=True)<br>  (key): Linear(in_features=768, out_features=768, bias=True)<br>  (value): Linear(in_features=768, out_features=768, bias=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertSelfOutput(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertIntermediate(<br>  (dense): Linear(in_features=768, out_features=3072, bias=True)<br>  (intermediate_act_fn): GELUActivation()<br>)\",\"Linear(in_features=768, out_features=3072, bias=True)\",\"GELUActivation()\",\"BertOutput(<br>  (dense): Linear(in_features=3072, out_features=768, bias=True)<br>  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)<br>  (dropout): Dropout(p=0.1, inplace=False)<br>)\",\"Linear(in_features=3072, out_features=768, bias=True)\",\"LayerNorm((768,), eps=1e-12, elementwise_affine=True)\",\"Dropout(p=0.1, inplace=False)\",\"BertPooler(<br>  (dense): Linear(in_features=768, out_features=768, bias=True)<br>  (activation): Tanh()<br>)\",\"Linear(in_features=768, out_features=768, bias=True)\",\"Tanh()\"],\"values\":[109482240,23837184,23440896,393216,1536,1536,0,85054464,85054464,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,590592,590592,0],\"type\":\"treemap\",\"root\":{\"color\":\"lightgrey\"}}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('d3bb13b7-4a14-422a-8df2-5520ccce4191');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fig = go.Figure(go.Sunburst(\n",
        "    labels = labels,\n",
        "    parents = parents,\n",
        "    values = values,\n",
        "))\n",
        "\n",
        "fig.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "351sS8BUcEw_",
        "outputId": "c07fab14-070d-4647-91fd-8281b6880ab3"
      },
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"e9cdb30e-0e8e-4875-9582-35c29b608db3\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e9cdb30e-0e8e-4875-9582-35c29b608db3\")) {                    Plotly.newPlot(                        \"e9cdb30e-0e8e-4875-9582-35c29b608db3\",                        [{\"labels\":[\"/BertModel\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings/Embedding\",\"/BertModel/BertEmbeddings/Embedding1\",\"/BertModel/BertEmbeddings/Embedding2\",\"/BertModel/BertEmbeddings/LayerNorm\",\"/BertModel/BertEmbeddings/Dropout\",\"/BertModel/BertEncoder\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer3\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer4\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer5\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer6\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer7\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer8\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer9\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer10\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer11\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention/Linear1\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention/Linear2\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput/Dropout\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate/GELUActivation\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput/Linear\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput/LayerNorm\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput/Dropout\",\"/BertModel/BertPooler\",\"/BertModel/BertPooler/Linear\",\"/BertModel/BertPooler/Tanh\"],\"parents\":[\"\",\"/BertModel\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings\",\"/BertModel/BertEmbeddings\",\"/BertModel\",\"/BertModel/BertEncoder\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer1\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer1/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer2\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer2/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer3\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer3\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer3/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer4\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer4\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer4/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer5\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer5\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer5/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer6\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer6\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer6/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer7\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer7\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer7/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer8\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer8\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer8/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer9\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer9\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer9/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer10\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer10\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer10/BertOutput\",\"/BertModel/BertEncoder/ModuleList\",\"/BertModel/BertEncoder/ModuleList/BertLayer11\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertAttention/BertSelfOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertIntermediate\",\"/BertModel/BertEncoder/ModuleList/BertLayer11\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput\",\"/BertModel/BertEncoder/ModuleList/BertLayer11/BertOutput\",\"/BertModel\",\"/BertModel/BertPooler\",\"/BertModel/BertPooler\"],\"values\":[109482240,23837184,23440896,393216,1536,1536,0,85054464,85054464,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,7087872,2363904,1771776,590592,590592,590592,0,592128,590592,1536,0,2362368,2362368,0,2361600,2360064,1536,0,590592,590592,0],\"type\":\"sunburst\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('e9cdb30e-0e8e-4875-9582-35c29b608db3');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Yc0xq2dfkMIf"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}