01.Lora微调.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "82b7f618-fc41-4bb9-a734-d27f78761def",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-07T09:41:04.833161Z",
     "iopub.status.busy": "2024-08-07T09:41:04.832828Z",
     "iopub.status.idle": "2024-08-07T09:41:12.913075Z",
     "shell.execute_reply": "2024-08-07T09:41:12.912518Z",
     "shell.execute_reply.started": "2024-08-07T09:41:04.833138Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2024-08-07 17:41:07.897768: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-08-07 17:41:07.909807: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-08-07 17:41:07.924154: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-08-07 17:41:07.928491: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-08-07 17:41:07.939288: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-08-07 17:41:08.882418: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from datasets import Dataset\n",
    "import pandas as pd\n",
    "from modelscope import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "from peft import LoraConfig, TaskType, get_peft_model\n",
    "from swanlab.integration.huggingface import SwanLabCallback\n",
    "import swanlab\n",
    "from transformers import DataCollatorForSeq2Seq, TrainingArguments, Trainer, GenerationConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bde8d52c-de3d-4779-8f15-cbc73728bca0",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-08-07T09:41:12.915455Z",
     "iopub.status.busy": "2024-08-07T09:41:12.914158Z",
     "iopub.status.idle": "2024-08-07T09:41:12.947307Z",
     "shell.execute_reply": "2024-08-07T09:41:12.946808Z",
     "shell.execute_reply.started": "2024-08-07T09:41:12.915428Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 将JSON文件转换为CSV文件\n",
    "# 将JSON文件转换为CSV文件\n",
    "# df = pd.read_json('./news_train.jsonl',lines = True)\n",
    "df = pd.read_csv('./data/xhsid20_shangpin_pinpai_1000.csv')\n",
    "# df['instruction'] = \"\"\"\n",
    "#             任务说明：\n",
    "#             你是一个文章品牌提取助手，你的任务是通过分析我提供的文章数据，提取文章中主要讲解的商品及商品品牌。\n",
    "#             请仔细阅读文章内容，并识别出文中提到的所有主要品牌名称及商品。请列出这些商品和品牌，该过程将分为两个主要步骤：\n",
    "#             1.判断是否为商品文章：\n",
    "#                 判断依据：文章是否直接介绍或推销某个商品。\n",
    "#                 若不满足该标准，则认为不是商品文章。\n",
    "#             2.对商品文章数据提取商品和品牌：\n",
    "#                 确保提取的品牌是文章中主要介绍的商品和品牌，输出形式为商品-品牌。\n",
    "\n",
    "#             提取规则\n",
    "#             文章数据识别: 主要通过对文章的主体对象判断是否直接介绍或推销某个商品而不是介绍非商品内容或者偶像组合。\n",
    "#             品牌识别： 需要准确判断文章中是否含有商品品牌。\n",
    "#             品牌完整性： 文章可能涉及多个商品品牌，确保输出品牌的完整性，用户名不作为商品品牌。\n",
    "#             门店名称提取： 如果文章为个体户门店，请确保能提取出门店名称。\n",
    "#             正确分类：主要分为三类，不是商品文章，有商品但无品牌，有品牌但无商品，有商品和品牌。\n",
    "#             根据品牌沿伸商品：对于文章中只有品牌没有商品的，则需要通过文章内容，获取最可能的商品\n",
    "\n",
    "#             输出标签格式如：\n",
    "#             若包含品牌则输出所有可能的品牌:[\"商品1-品牌名1\", \"商品2-品牌名2\", ...]。\n",
    "#             若商品文章数据不包含品牌则输出:[\"无商品品牌\"]。\n",
    "#             若不是商品文章数据则输出:[\"不是商品文章\"]。\n",
    "#             \"\"\"\n",
    "df['instruction'] = \"\"\"\n",
    "                         任务说明：\n",
    "             你是一个文章商品和品牌提取助手，你的任务是通过分析我提供的文章数据，提取文章中的商品及对应品牌。\n",
    "             请仔细阅读文章内容，并识别出文中提到的所有商品和品牌。请列出这些商品和品牌，该过程将分为两个主要步骤：\n",
    "             1.判断是否为商品文章：\n",
    "                 判断依据：文章是否直接介绍或推销某个商品。\n",
    "                 若不满足该标准，则认为不是商品文章，输出格式为[\"不是商品文章\"]。\n",
    "             2.对商品文章数据提取商品和品牌：\n",
    "                 分情况：若商品和品牌都存在，则输出格式为[\"商品名-品牌名\", \"商品名-品牌名\", ...]\n",
    "                        若只存在商品，则输出格式为[\"无商品品牌\"]\n",
    "                        若只存在品牌，则根据文章内容生成对应品牌的商品，输出格式为[\"生成的商品名-品牌名\", \"生成的商品名-品牌名\", ..., \"生成的商品名-品牌名\"]\n",
    "\n",
    "             提取规则\n",
    "             文章数据识别: 主要通过对文章的主体对象判断是否直接介绍或推销某个商品而不是介绍非商品内容或者偶像组合。\n",
    "             品牌识别： 需要准确判断文章中是否含有商品品牌。\n",
    "             品牌完整性： 文章可能涉及多个商品品牌，确保输出品牌的完整性，用户名不作为商品品牌。\n",
    "             门店名称提取： 如果文章为个体户门店，请确保能提取出门店名称。\n",
    "             \"\"\"\n",
    "df = df[['instruction','input', 'output']]\n",
    "ds = Dataset.from_pandas(df)\n",
    "test_df = df[50:110]  # 取前10条做主观评测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ff51ae32-be18-4584-b741-83bac1b6fd41",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-07T09:41:12.948196Z",
     "iopub.status.busy": "2024-08-07T09:41:12.948015Z",
     "iopub.status.idle": "2024-08-07T09:41:12.951771Z",
     "shell.execute_reply": "2024-08-07T09:41:12.951196Z",
     "shell.execute_reply.started": "2024-08-07T09:41:12.948177Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_df.to_json('./autodl-tmp/data/shangpinpinpai_test.json', orient='records',indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "17b5054a-8ef8-451c-ba56-1aab6ed6de10",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T07:34:20.601611Z",
     "iopub.status.busy": "2024-08-01T07:34:20.601407Z",
     "iopub.status.idle": "2024-08-01T07:34:20.609697Z",
     "shell.execute_reply": "2024-08-01T07:34:20.609319Z",
     "shell.execute_reply.started": "2024-08-01T07:34:20.601595Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>以前为了怕卡粉厚重 底妆一直追求轻薄 结果遮瑕又弱妆效也一般,这次用zati粉底液就多涂了一...</td>\n",
       "      <td>[“粉底液-zati\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td># 吗喽 # 动物 # 可爱</td>\n",
       "      <td>[“不是商品文章”]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>威飒的黑色内饰提供经典、时尚的外观，并且耐脏，无需勤打理。座舱采用T字型对称式布局，视觉上宽...</td>\n",
       "      <td>[“内饰-威飒\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>1、aisei珍珠米白，很通透的大直径玻璃珠，日常戴不容易出错\\n2、aisei月见茶灰，清...</td>\n",
       "      <td>[“美瞳-aisei\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>😱谁家电视这么敢的啊！？\\n🔥海信E8N  Ultra限时免费安装📍新品首发强势来袭！\\n⭕...</td>\n",
       "      <td>[“电视-海信\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>摩拉尤品这个新品水牛奶[哇R]好喝得像液体奶酪!!\\n[自拍R]烫金礼盒装，春天的配色，颜值...</td>\n",
       "      <td>[“水牛奶-摩拉尤品\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>海口现货❤️100ml到手3360❤️50ml到手1860❤️日期新鲜哟❤️送人🈶礼袋，礼盒...</td>\n",
       "      <td>[“不是商品文章”]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>💖•新店入驻💗\\n🥳霸王茶姬 化州首店🤗\\n[咖啡]打造东⽅新茶铺💕\\n⏰即将开业 敬请期待🎉</td>\n",
       "      <td>[“奶茶-霸王茶姬\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>摩圈、MOTO图库、公众号、抖音搜索:谷GU追焦 [doge]</td>\n",
       "      <td>[“不是商品文章”]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>原价出两张5.18惠州dearjane\\n本来5.16要去福州看周杰伦 然后5.17要去厦门...</td>\n",
       "      <td>[“不是商品文章”]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>#国货[话题]##315消费者权益[话题]##娃哈哈[话题]#</td>\n",
       "      <td>[“饮料-娃哈哈\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>听说紫色黑8爆单了\\n不愧是stussy\\n大家抢购速度真的很快～\\n爱你们～</td>\n",
       "      <td>[“衣服-stussy\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>2016年福特福克斯两厢\\n125手动挡风尚版、个人私家一手车、车况好实表7万公里、天窗、尾...</td>\n",
       "      <td>[“车-福特\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>安妮家绝绝子一套[萌萌哒R]</td>\n",
       "      <td>[“套装-安妮家\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>#郓城网红婚宴酒店[话题]# #郓城婚礼[话题]# #郓城天成酒店[话题]#</td>\n",
       "      <td>[“不是商品文章”]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>#六福珠宝[话题]#【点钻系列]\\n金，永恒不变，穿越千年。\\n钻，时尚前沿，闪耀新鲜。\\n...</td>\n",
       "      <td>[“珠宝-六福珠宝\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>[清单R]今天咱们来讲下，湖南户籍线上开具无犯罪记录证明的方法。\\n[一R]：baidu搜索...</td>\n",
       "      <td>[“不是商品文章”]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>清水寺每天关门的时间是下午6点，所以要去的朋友圈要注意一下时间，我感觉最好去的时间是早上或者...</td>\n",
       "      <td>[“美食-鲷寿司”]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>苹果原装正品magsafe外接电源\\n没用过几次，挺新。\\n现在已经停产。\\n低于5张的别来...</td>\n",
       "      <td>[“手机-苹果\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>\\n                         任务说明：\\n            ...</td>\n",
       "      <td>两周前在digiDirect订的富士XT30-1855套机，当时说要等四到八周，没想到今天就...</td>\n",
       "      <td>[“相机-富士\"]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           instruction  \\\n",
       "90   \\n                         任务说明：\\n            ...   \n",
       "91   \\n                         任务说明：\\n            ...   \n",
       "92   \\n                         任务说明：\\n            ...   \n",
       "93   \\n                         任务说明：\\n            ...   \n",
       "94   \\n                         任务说明：\\n            ...   \n",
       "95   \\n                         任务说明：\\n            ...   \n",
       "96   \\n                         任务说明：\\n            ...   \n",
       "97   \\n                         任务说明：\\n            ...   \n",
       "98   \\n                         任务说明：\\n            ...   \n",
       "99   \\n                         任务说明：\\n            ...   \n",
       "100  \\n                         任务说明：\\n            ...   \n",
       "101  \\n                         任务说明：\\n            ...   \n",
       "102  \\n                         任务说明：\\n            ...   \n",
       "103  \\n                         任务说明：\\n            ...   \n",
       "104  \\n                         任务说明：\\n            ...   \n",
       "105  \\n                         任务说明：\\n            ...   \n",
       "106  \\n                         任务说明：\\n            ...   \n",
       "107  \\n                         任务说明：\\n            ...   \n",
       "108  \\n                         任务说明：\\n            ...   \n",
       "109  \\n                         任务说明：\\n            ...   \n",
       "\n",
       "                                                 input         output  \n",
       "90   以前为了怕卡粉厚重 底妆一直追求轻薄 结果遮瑕又弱妆效也一般,这次用zati粉底液就多涂了一...   [“粉底液-zati\"]  \n",
       "91                                      # 吗喽 # 动物 # 可爱     [“不是商品文章”]  \n",
       "92   威飒的黑色内饰提供经典、时尚的外观，并且耐脏，无需勤打理。座舱采用T字型对称式布局，视觉上宽...      [“内饰-威飒\"]  \n",
       "93   1、aisei珍珠米白，很通透的大直径玻璃珠，日常戴不容易出错\\n2、aisei月见茶灰，清...   [“美瞳-aisei\"]  \n",
       "94   😱谁家电视这么敢的啊！？\\n🔥海信E8N  Ultra限时免费安装📍新品首发强势来袭！\\n⭕...      [“电视-海信\"]  \n",
       "95   摩拉尤品这个新品水牛奶[哇R]好喝得像液体奶酪!!\\n[自拍R]烫金礼盒装，春天的配色，颜值...   [“水牛奶-摩拉尤品\"]  \n",
       "96   海口现货❤️100ml到手3360❤️50ml到手1860❤️日期新鲜哟❤️送人🈶礼袋，礼盒...     [“不是商品文章”]  \n",
       "97     💖•新店入驻💗\\n🥳霸王茶姬 化州首店🤗\\n[咖啡]打造东⽅新茶铺💕\\n⏰即将开业 敬请期待🎉    [“奶茶-霸王茶姬\"]  \n",
       "98                     摩圈、MOTO图库、公众号、抖音搜索:谷GU追焦 [doge]     [“不是商品文章”]  \n",
       "99   原价出两张5.18惠州dearjane\\n本来5.16要去福州看周杰伦 然后5.17要去厦门...     [“不是商品文章”]  \n",
       "100                    #国货[话题]##315消费者权益[话题]##娃哈哈[话题]#     [“饮料-娃哈哈\"]  \n",
       "101            听说紫色黑8爆单了\\n不愧是stussy\\n大家抢购速度真的很快～\\n爱你们～  [“衣服-stussy\"]  \n",
       "102  2016年福特福克斯两厢\\n125手动挡风尚版、个人私家一手车、车况好实表7万公里、天窗、尾...       [“车-福特\"]  \n",
       "103                                     安妮家绝绝子一套[萌萌哒R]     [“套装-安妮家\"]  \n",
       "104             #郓城网红婚宴酒店[话题]# #郓城婚礼[话题]# #郓城天成酒店[话题]#     [“不是商品文章”]  \n",
       "105  #六福珠宝[话题]#【点钻系列]\\n金，永恒不变，穿越千年。\\n钻，时尚前沿，闪耀新鲜。\\n...    [“珠宝-六福珠宝\"]  \n",
       "106  [清单R]今天咱们来讲下，湖南户籍线上开具无犯罪记录证明的方法。\\n[一R]：baidu搜索...     [“不是商品文章”]  \n",
       "107  清水寺每天关门的时间是下午6点，所以要去的朋友圈要注意一下时间，我感觉最好去的时间是早上或者...     [“美食-鲷寿司”]  \n",
       "108  苹果原装正品magsafe外接电源\\n没用过几次，挺新。\\n现在已经停产。\\n低于5张的别来...      [“手机-苹果\"]  \n",
       "109  两周前在digiDirect订的富士XT30-1855套机，当时说要等四到八周，没想到今天就...      [“相机-富士\"]  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d99a9edf-6563-4d9d-a03b-48fb083ac00f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:31.620608Z",
     "iopub.status.busy": "2024-08-01T06:59:31.620301Z",
     "iopub.status.idle": "2024-08-01T06:59:31.628547Z",
     "shell.execute_reply": "2024-08-01T06:59:31.627933Z",
     "shell.execute_reply.started": "2024-08-01T06:59:31.620589Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 980 entries, 0 to 979\n",
      "Data columns (total 3 columns):\n",
      " #   Column       Non-Null Count  Dtype \n",
      "---  ------       --------------  ----- \n",
      " 0   instruction  980 non-null    object\n",
      " 1   input        980 non-null    object\n",
      " 2   output       980 non-null    object\n",
      "dtypes: object(3)\n",
      "memory usage: 23.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b8d92e84-db78-40a6-804c-cc694b1f4613",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:34.698903Z",
     "iopub.status.busy": "2024-08-01T06:59:34.698594Z",
     "iopub.status.idle": "2024-08-01T06:59:34.705056Z",
     "shell.execute_reply": "2024-08-01T06:59:34.704467Z",
     "shell.execute_reply.started": "2024-08-01T06:59:34.698883Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [instruction, input, output]\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['output'].isnull()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e72d646f-1d9b-4c70-89fa-9ad43b73d770",
   "metadata": {},
   "source": [
    "# 处理数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d2b25e3b-6a38-4c00-a94e-939b66489ee8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:36.704109Z",
     "iopub.status.busy": "2024-08-01T06:59:36.703787Z",
     "iopub.status.idle": "2024-08-01T06:59:37.610553Z",
     "shell.execute_reply": "2024-08-01T06:59:37.610015Z",
     "shell.execute_reply.started": "2024-08-01T06:59:36.704089Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Qwen2Tokenizer(name_or_path='/mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct', vocab_size=151643, model_max_length=131072, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'eos_token': '<|im_end|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>']}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
       "\t151643: AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t151644: AddedToken(\"<|im_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t151645: AddedToken(\"<|im_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2-7B-Instruct', use_fast=False, trust_remote_code=True)\n",
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "89c283b8-8efd-4be3-80df-5c73c2e94860",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:37.611789Z",
     "iopub.status.busy": "2024-08-01T06:59:37.611453Z",
     "iopub.status.idle": "2024-08-01T06:59:37.619542Z",
     "shell.execute_reply": "2024-08-01T06:59:37.619019Z",
     "shell.execute_reply.started": "2024-08-01T06:59:37.611771Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def process_func(example):\n",
    "    MAX_LENGTH = 384    # Llama分词器会将一个中文字切分为多个token，因此需要放开一些最大长度，保证数据的完整性\n",
    "    input_ids, attention_mask, labels = [], [], []\n",
    "    instruction = tokenizer(f\"<|im_start|>system\\n{example['instruction']}<|im_end|>\\n<|im_start|>user\\n{example['input']}<|im_end|>\\n<|im_start|>assistant\\n\", add_special_tokens=False)  # add_special_tokens 不在开头加 special_tokens\n",
    "    response = tokenizer(f\"{example['output']}\", add_special_tokens=False)\n",
    "    input_ids = instruction[\"input_ids\"] + response[\"input_ids\"] + [tokenizer.pad_token_id]\n",
    "    attention_mask = instruction[\"attention_mask\"] + response[\"attention_mask\"] + [1]  # 因为eos token咱们也是要关注的所以 补充为1\n",
    "    labels = [-100] * len(instruction[\"input_ids\"]) + response[\"input_ids\"] + [tokenizer.pad_token_id]  \n",
    "    if len(input_ids) > MAX_LENGTH:  # 做一个截断\n",
    "        input_ids = input_ids[:MAX_LENGTH]\n",
    "        attention_mask = attention_mask[:MAX_LENGTH]\n",
    "        labels = labels[:MAX_LENGTH]\n",
    "    return {\n",
    "        \"input_ids\": input_ids,\n",
    "        \"attention_mask\": attention_mask,\n",
    "        \"labels\": labels\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "880385ca-3de6-44f9-9d6d-8a84095f0619",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:40.540675Z",
     "iopub.status.busy": "2024-08-01T06:59:40.540131Z",
     "iopub.status.idle": "2024-08-01T06:59:47.216995Z",
     "shell.execute_reply": "2024-08-01T06:59:47.216559Z",
     "shell.execute_reply.started": "2024-08-01T06:59:40.540654Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 980/980 [00:03<00:00, 245.60 examples/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask', 'labels'],\n",
       "    num_rows: 980\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_id = ds.map(process_func, remove_columns=ds.column_names)\n",
    "tokenized_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5eb843d1-f781-4a8a-8fe4-f9877884b85d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:47.218175Z",
     "iopub.status.busy": "2024-08-01T06:59:47.217844Z",
     "iopub.status.idle": "2024-08-01T06:59:47.223952Z",
     "shell.execute_reply": "2024-08-01T06:59:47.223556Z",
     "shell.execute_reply.started": "2024-08-01T06:59:47.218151Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<|im_start|>system\\n\\n                         任务说明：\\n             你是一个文章商品和品牌提取助手，你的任务是通过分析我提供的文章数据，提取文章中的商品及对应品牌。\\n             请仔细阅读文章内容，并识别出文中提到的所有商品和品牌。请列出这些商品和品牌，该过程将分为两个主要步骤：\\n             1.判断是否为商品文章：\\n                 判断依据：文章是否直接介绍或推销某个商品。\\n                 若不满足该标准，则认为不是商品文章，输出格式为[\"不是商品文章\"]。\\n             2.对商品文章数据提取商品和品牌：\\n                 分情况：若商品和品牌都存在，则输出格式为[\"商品名-品牌名\", \"商品名-品牌名\", ...]\\n                        若只存在商品，则输出格式为[\"无商品品牌\"]\\n                        若只存在品牌，则根据文章内容生成对应品牌的商品，输出格式为[\"生成的商品名-品牌名\", \"生成的商品名-品牌名\", ..., \"生成的商品名-品牌名\"]\\n\\n             提取规则\\n             文章数据识别: 主要通过对文章的主体对象判断是否直接介绍或推销某个商品而不是介绍非商品内容或者偶像组合。\\n             品牌识别： 需要准确判断文章中是否含有商品品牌。\\n             品牌完整性： 文章可能涉及多个商品品牌，确保输出品牌的完整性，用户名不作为商品品牌。\\n             门店名称提取： 如果文章为个体户门店，请确保能提取出门店名称。\\n             <|im_end|>\\n<|im_start|>user\\n上次护肤Roution分享了褒曼2号，还剩三分之一，续了Acaderma的平衡稳健水，看图片用量我还在磨合哈哈，现在换季购入了purid的舒润露，这篇详细说下感受：\\n一\\n洁面完用护肤水时通常'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(tokenized_id[0]['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d040f269-93ba-44c6-8758-692cc83adc2b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:47.224785Z",
     "iopub.status.busy": "2024-08-01T06:59:47.224638Z",
     "iopub.status.idle": "2024-08-01T06:59:47.228994Z",
     "shell.execute_reply": "2024-08-01T06:59:47.228557Z",
     "shell.execute_reply.started": "2024-08-01T06:59:47.224770Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(list(filter(lambda x: x != -100, tokenized_id[1][\"labels\"])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d47b109d-92e7-48c3-a396-2aa8f272da2c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:47.230127Z",
     "iopub.status.busy": "2024-08-01T06:59:47.229835Z",
     "iopub.status.idle": "2024-08-01T06:59:47.233975Z",
     "shell.execute_reply": "2024-08-01T06:59:47.233553Z",
     "shell.execute_reply.started": "2024-08-01T06:59:47.230110Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "int"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(tokenized_id[0]['input_ids'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cb25004-42b3-4001-9fb7-81cea1a11ec0",
   "metadata": {},
   "source": [
    "# 创建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ff947173-1773-40da-80ab-34eaaaaafdf0",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T06:59:49.838089Z",
     "iopub.status.busy": "2024-08-01T06:59:49.837771Z",
     "iopub.status.idle": "2024-08-01T07:01:22.019605Z",
     "shell.execute_reply": "2024-08-01T07:01:22.019127Z",
     "shell.execute_reply.started": "2024-08-01T06:59:49.838067Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 4/4 [01:24<00:00, 21.16s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Qwen2ForCausalLM(\n",
       "  (model): Qwen2Model(\n",
       "    (embed_tokens): Embedding(152064, 3584)\n",
       "    (layers): ModuleList(\n",
       "      (0-27): 28 x Qwen2DecoderLayer(\n",
       "        (self_attn): Qwen2SdpaAttention(\n",
       "          (q_proj): Linear(in_features=3584, out_features=3584, bias=True)\n",
       "          (k_proj): Linear(in_features=3584, out_features=512, bias=True)\n",
       "          (v_proj): Linear(in_features=3584, out_features=512, bias=True)\n",
       "          (o_proj): Linear(in_features=3584, out_features=3584, bias=False)\n",
       "          (rotary_emb): Qwen2RotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Qwen2MLP(\n",
       "          (gate_proj): Linear(in_features=3584, out_features=18944, bias=False)\n",
       "          (up_proj): Linear(in_features=3584, out_features=18944, bias=False)\n",
       "          (down_proj): Linear(in_features=18944, out_features=3584, bias=False)\n",
       "          (act_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Qwen2RMSNorm()\n",
       "        (post_attention_layernorm): Qwen2RMSNorm()\n",
       "      )\n",
       "    )\n",
       "    (norm): Qwen2RMSNorm()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=3584, out_features=152064, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = \"cuda\"\n",
    "model = AutoModelForCausalLM.from_pretrained('Qwen/Qwen2-7B-Instruct',device_map=\"auto\",torch_dtype=torch.bfloat16)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "debd0edf-8cd3-4c91-9f68-502c0a05d716",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.020782Z",
     "iopub.status.busy": "2024-08-01T07:01:22.020473Z",
     "iopub.status.idle": "2024-08-01T07:01:22.023535Z",
     "shell.execute_reply": "2024-08-01T07:01:22.023134Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.020765Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.enable_input_require_grads()  # 开启梯度检查点时，要执行该方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cb621331-cebd-4c19-a47c-885b398b3fc0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.024272Z",
     "iopub.status.busy": "2024-08-01T07:01:22.024059Z",
     "iopub.status.idle": "2024-08-01T07:01:22.027525Z",
     "shell.execute_reply": "2024-08-01T07:01:22.027123Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.024258Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.bfloat16"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2a92d9f-0ed0-4869-b299-ac5b4cf31956",
   "metadata": {},
   "source": [
    "# Lora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4bc0c2da-f10a-42e4-a1f0-448c3487a26a",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.028758Z",
     "iopub.status.busy": "2024-08-01T07:01:22.028495Z",
     "iopub.status.idle": "2024-08-01T07:01:22.031577Z",
     "shell.execute_reply": "2024-08-01T07:01:22.031123Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.028742Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "config = LoraConfig(\n",
    "    task_type=TaskType.CAUSAL_LM,  # 传统语言模型\n",
    "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n",
    "    inference_mode=False,  # 训练模式\n",
    "    r=8,  # Lora 秩\n",
    "    lora_alpha=32,  # Lora alaph，具体作用参见 Lora 原理\n",
    "    lora_dropout=0.1  # Dropout 比例\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "499ff466-1068-4fd0-bfbb-5ef036ca26bc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.032378Z",
     "iopub.status.busy": "2024-08-01T07:01:22.032109Z",
     "iopub.status.idle": "2024-08-01T07:01:22.035506Z",
     "shell.execute_reply": "2024-08-01T07:01:22.035122Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.032363Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LoraConfig(peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path=None, revision=None, task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>, inference_mode=False, r=8, target_modules={'q_proj', 'up_proj', 'down_proj', 'gate_proj', 'o_proj', 'k_proj', 'v_proj'}, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', loftq_config={}, use_dora=False, layer_replication=None)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "98590d51-7dad-46ae-8397-52e14ec539af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.036293Z",
     "iopub.status.busy": "2024-08-01T07:01:22.036013Z",
     "iopub.status.idle": "2024-08-01T07:01:22.447520Z",
     "shell.execute_reply": "2024-08-01T07:01:22.447064Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.036271Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LoraConfig(peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path='/mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct', revision=None, task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>, inference_mode=False, r=8, target_modules={'q_proj', 'up_proj', 'down_proj', 'gate_proj', 'o_proj', 'k_proj', 'v_proj'}, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', loftq_config={}, use_dora=False, layer_replication=None)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = get_peft_model(model, config)\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "793cb158-4de4-4d19-8855-867c72e3e9b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.448554Z",
     "iopub.status.busy": "2024-08-01T07:01:22.448177Z",
     "iopub.status.idle": "2024-08-01T07:01:22.455482Z",
     "shell.execute_reply": "2024-08-01T07:01:22.455059Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.448536Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 20,185,088 || all params: 7,635,801,600 || trainable%: 0.2643\n"
     ]
    }
   ],
   "source": [
    "model.print_trainable_parameters()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7278d37-fb34-4d10-9448-581e49bb1eeb",
   "metadata": {},
   "source": [
    "# 配置训练参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3da4fa9a-b93e-4dc4-ad28-640d056a34a5",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.456298Z",
     "iopub.status.busy": "2024-08-01T07:01:22.456015Z",
     "iopub.status.idle": "2024-08-01T07:01:22.469441Z",
     "shell.execute_reply": "2024-08-01T07:01:22.469054Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.456282Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=\"./Qwen/Qwen2_pinpai_1000\",\n",
    "    logging_dir='./Qwen/logs',\n",
    "    logging_steps=1,\n",
    "    per_device_train_batch_size=4,\n",
    "    gradient_accumulation_steps=4,\n",
    "    num_train_epochs=5,\n",
    "    save_steps=10,  # 为了快速演示，这里设置10，建议你设置成100\n",
    "    learning_rate=1e-4,\n",
    "    save_on_each_node=True,\n",
    "    gradient_checkpointing=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "35b4fc3e-bf1a-4693-ad65-c118d09e9968",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.470084Z",
     "iopub.status.busy": "2024-08-01T07:01:22.469925Z",
     "iopub.status.idle": "2024-08-01T07:01:22.473469Z",
     "shell.execute_reply": "2024-08-01T07:01:22.473058Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.470069Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "swanlab_callback = SwanLabCallback(\n",
    "    project=\"Qwen2-fintune\",\n",
    "    experiment_name=\"Qwen2-7B-Instruct-pinpai_1000\",\n",
    "    description=\"使用通义千问Qwen2-7B-Instruct模型在pinpai1000上prompt微调。\",\n",
    "    config={\n",
    "        \"model\": \"Qwen/Qwen2-7B-Instruct\",\n",
    "        \"dataset\": \"xhspinpai.json\",\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1f4b968e-68a4-42c3-8bf7-837cab759659",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T07:01:22.474903Z",
     "iopub.status.busy": "2024-08-01T07:01:22.474606Z",
     "iopub.status.idle": "2024-08-01T07:32:23.007508Z",
     "shell.execute_reply": "2024-08-01T07:32:23.007199Z",
     "shell.execute_reply.started": "2024-08-01T07:01:22.474888Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Detected kernel version 4.19.24, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-08-01 15:01:22,909] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n",
      "\u001b[93m [WARNING] \u001b[0m Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH\n",
      "\u001b[93m [WARNING] \u001b[0m sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3\n",
      "\u001b[93m [WARNING] \u001b[0m please install triton==1.0.0 if you want to use sparse attention\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: Tracking run with swanlab version 0.3.9                                   \n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: Run data will be saved locally in \u001b[35m\u001b[1m/mnt/workspace/swanlog/run-20240801_150126-a3b1799d\u001b[0m\u001b[0m\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 👋 Hi \u001b[1m\u001b[39mzs-train\u001b[0m\u001b[0m, welcome to swanlab!\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: Syncing run \u001b[33mQwen2-7B-Instruct-pinpai_1000_Aug01_15-01-26\u001b[0m to the cloud\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🌟 Run `\u001b[1mswanlab watch -l /mnt/workspace/swanlog\u001b[0m` to view SwanLab Experiment Dashboard locally\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🏠 View project at \u001b[34m\u001b[4mhttps://swanlab.cn/@zs-train/Qwen2-fintune\u001b[0m\u001b[0m\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://swanlab.cn/@zs-train/Qwen2-fintune/runs/1ak1fest0vzqqzvr2jab7\u001b[0m\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<!DOCTYPE html>\n",
       "<html lang=\"en\">\n",
       "<head>\n",
       "    <meta charset=\"UTF-8\">\n",
       "    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n",
       "    <title>Show Iframe</title>\n",
       "    \n",
       "        <script>\n",
       "            function showIframe() {\n",
       "                var iframeHtml = '<iframe src=\"https://swanlab.cn/@zs-train/Qwen2-fintune/runs/1ak1fest0vzqqzvr2jab7\" width=100% height=\"600\" frameborder=\"no\"></iframe>';\n",
       "                document.getElementById('iframeContainer').innerHTML = iframeHtml;\n",
       "            }\n",
       "        </script>\n",
       "        \n",
       "</head>\n",
       "<body>\n",
       "    <style>\n",
       "        .interactive-button {\n",
       "            display: flex;\n",
       "            align-items: center;\n",
       "            height: 36px;\n",
       "            border: 0px;\n",
       "            background-color: #2c8f63;\n",
       "            color: white;\n",
       "            padding: 10px 20px;\n",
       "            transition: background-color 0.3s, transform 0.2s;\n",
       "        }\n",
       "\n",
       "        .interactive-button:hover {\n",
       "            background-color: #5cab87;\n",
       "            cursor: pointer;\n",
       "        }\n",
       "\n",
       "        .interactive-button:active {\n",
       "            background-color: #217952;\n",
       "            transform: scale(0.96);\n",
       "        }\n",
       "    </style>\n",
       "    <br>\n",
       "    <button onclick=\"showIframe()\" class=\"interactive-button\">\n",
       "        <svg style=\"height: 16px; margin-right: 8px;\" viewBox=\"0 0 46 46\" fill=\"none\">\n",
       "            <path\n",
       "                d=\"M10.8439 21.1974C10.6414 21.2854 10.4477 21.3925 10.2655 21.5173L10.2069 21.5652C10.1839 21.58 10.1625 21.5969 10.1429 21.6159C6.29135 24.6118 4.22831 29.4416 5.32646 34.282C5.94656 37.0577 7.50461 39.5348 9.73801 41.2958C11.9714 43.0568 14.7436 43.994 17.5874 43.9495H18.0219C19.8864 43.8697 21.7087 43.3694 23.3526 42.486C24.9964 41.6026 26.4193 40.3589 27.5147 38.848C28.61 37.3371 29.3496 35.598 29.678 33.761C30.0065 31.9239 29.9153 30.0363 29.4112 28.2395C28.9181 26.4723 27.8919 24.8437 26.9937 23.2551C25.4158 20.4653 23.8343 17.6764 22.2492 14.8884C21.7801 14.0647 21.3057 13.2465 20.8419 12.4228C20.2315 11.3353 19.2746 10.1519 19.224 8.86183C19.1733 7.57176 20.2235 6.32701 21.5082 6.07912C23.9284 5.61801 25.0639 8.24078 25.0693 8.23812C25.363 8.94035 25.9123 9.50489 26.6063 9.81764C27.3002 10.1304 28.087 10.168 28.8077 9.92298C29.5283 9.67791 30.1291 9.1684 30.4885 8.49743C30.8479 7.82646 30.9392 7.04405 30.7439 6.30835C30.1514 4.37314 28.9133 2.69953 27.2363 1.56656C25.7615 0.511704 23.9847 -0.0372109 22.1719 0.00195984C20.9049 0.00893199 19.6532 0.27989 18.4967 0.797557C17.3402 1.31522 16.3043 2.06823 15.4551 3.00856C14.49 4.08707 13.7984 5.38193 13.4389 6.78385C13.0794 8.18576 13.0624 9.6536 13.3894 11.0635C13.52 11.593 13.6984 12.1095 13.9225 12.6067C14.5595 14.0514 15.4951 15.3681 16.284 16.7355C17.2525 18.4147 18.2209 20.0948 19.1893 21.7758C20.1578 23.4568 21.1351 25.1449 22.1213 26.8401C22.9209 28.2421 23.7925 29.4682 23.8805 31.1528C23.9175 32.0513 23.7682 32.9479 23.4419 33.7859C23.1156 34.6239 22.6194 35.3854 21.9845 36.0223C21.3496 36.6592 20.5897 37.1578 19.7527 37.4868C18.9157 37.8157 18.0196 37.9678 17.121 37.9336C14.0024 37.7923 11.6488 35.4814 11.1744 32.4588C10.58 28.6419 13.552 26.5469 13.552 26.5469C14.1782 26.1785 14.6497 25.5955 14.8791 24.906C15.1084 24.2166 15.0801 23.4673 14.7993 22.7971C14.5186 22.127 14.0044 21.5813 13.3521 21.2611C12.6998 20.941 11.9536 20.8682 11.2517 21.0561C11.1174 21.0939 10.9856 21.1402 10.8572 21.1947\"\n",
       "                fill=\"white\"\n",
       "            />\n",
       "            <path\n",
       "                d=\"M42.8101 31.5968C42.8109 30.5198 42.7218 29.4445 42.5435 28.3823C42.2663 26.7069 41.7464 25.0808 41.0002 23.5552C40.5524 22.6463 39.9874 21.7374 39.1024 21.2417C38.6593 20.9919 38.1589 20.8617 37.6502 20.8639C37.1416 20.8661 36.6423 21.0006 36.2013 21.2541C35.7604 21.5077 35.393 21.8716 35.1352 22.3101C34.8775 22.7485 34.7382 23.2466 34.7312 23.7552C34.7072 24.8773 35.3149 25.8875 35.768 26.9217C36.5212 28.6453 36.8623 30.5208 36.7642 32.3993C36.6661 34.2777 36.1315 36.1075 35.2029 37.7433C35.146 37.8404 35.0952 37.941 35.051 38.0445C34.8623 38.4842 34.7635 38.9573 34.7605 39.4358C34.7802 40.1222 35.0356 40.7808 35.4835 41.3011C35.9315 41.8214 36.5449 42.1717 37.2207 42.2932C38.8759 42.589 40.1899 41.347 40.8856 39.9609C42.1643 37.3589 42.823 34.4961 42.8101 31.5968Z\"\n",
       "                fill=\"white\"\n",
       "            />\n",
       "            <path\n",
       "                d=\"M28.2309 11.8938C28.1761 11.9043 28.1218 11.9176 28.0683 11.9338C27.9593 11.9642 27.8611 12.0249 27.7851 12.1088C27.7091 12.1928 27.6584 12.2965 27.6389 12.408C27.6193 12.5195 27.6318 12.6343 27.6748 12.7391C27.7178 12.8438 27.7895 12.9343 27.8818 12.9999C29.2375 14.0252 30.3809 15.3043 31.2482 16.7662C31.4838 17.1677 31.6888 17.5865 31.8612 18.0189C32.0052 18.3921 32.1971 18.8799 32.6822 18.8532C33.0607 18.8346 33.2153 18.512 33.3192 18.1895C33.8137 16.5125 33.9678 14.7534 33.7723 13.0159C33.6331 12.0693 33.4155 11.1359 33.122 10.2252C33.0775 10.0047 32.9744 9.80029 32.8235 9.6335C32.7273 9.54627 32.6054 9.49262 32.4761 9.4806C32.3468 9.46859 32.2171 9.49886 32.1065 9.56687C32.0016 9.65188 31.9115 9.75365 31.8399 9.86806C31.3956 10.4658 30.825 10.9581 30.1687 11.3101C29.8377 11.4861 29.4893 11.6272 29.1292 11.7312C28.828 11.8192 28.5215 11.8325 28.2309 11.8938Z\"\n",
       "                fill=\"white\"\n",
       "            />\n",
       "        </svg>\n",
       "        Display SwanLab Board\n",
       "    </button>\n",
       "    <br>\n",
       "    <div id=\"iframeContainer\"></div>\n",
       "</body>\n",
       "</html>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='305' max='305' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [305/305 30:44, Epoch 4/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.627200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5.033000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.342800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.002400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2.020200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.839900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.741900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.992600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>1.027800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.027500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.378900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.957800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.826000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.593300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.292000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.349700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.824300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.389800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.207100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.676000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.290900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.175300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.211000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.585300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.597400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.755000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>0.091700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.444200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.400900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.289700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>1.306600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.554500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.367600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.628400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.453400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>1.638300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.546200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.244100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.533400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.417000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>0.243300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>0.231800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>0.272400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>0.532500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>0.764200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>0.286100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>0.290900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>0.374100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>0.010900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.290000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>0.318100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>0.552200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>0.431100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>0.291400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>0.395300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>0.268900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>0.292500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>0.578800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>0.321700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>0.448800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61</td>\n",
       "      <td>1.568500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62</td>\n",
       "      <td>0.147500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63</td>\n",
       "      <td>0.226300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>64</td>\n",
       "      <td>0.400100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>65</td>\n",
       "      <td>0.117200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66</td>\n",
       "      <td>0.255800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67</td>\n",
       "      <td>0.208000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>68</td>\n",
       "      <td>0.158800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69</td>\n",
       "      <td>0.192400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>0.080900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71</td>\n",
       "      <td>0.240900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>0.115300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>0.040000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>0.177000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>0.108500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>76</td>\n",
       "      <td>0.385300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>77</td>\n",
       "      <td>0.261400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78</td>\n",
       "      <td>0.134400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>79</td>\n",
       "      <td>0.489500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>0.539500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>81</td>\n",
       "      <td>0.175600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>82</td>\n",
       "      <td>0.339500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>83</td>\n",
       "      <td>0.087000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>84</td>\n",
       "      <td>0.230200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85</td>\n",
       "      <td>0.452800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>86</td>\n",
       "      <td>0.189200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>87</td>\n",
       "      <td>0.029500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>88</td>\n",
       "      <td>0.283800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>89</td>\n",
       "      <td>0.142700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>0.350300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>91</td>\n",
       "      <td>0.250600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>92</td>\n",
       "      <td>0.181100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>93</td>\n",
       "      <td>0.129300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>94</td>\n",
       "      <td>0.125600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95</td>\n",
       "      <td>0.258900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>0.231400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>97</td>\n",
       "      <td>0.212500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>98</td>\n",
       "      <td>0.252800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>0.079400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.492500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>101</td>\n",
       "      <td>0.356400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>102</td>\n",
       "      <td>0.120900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>103</td>\n",
       "      <td>0.252300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>104</td>\n",
       "      <td>0.084300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>105</td>\n",
       "      <td>0.055800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>106</td>\n",
       "      <td>0.200300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>107</td>\n",
       "      <td>0.257700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>108</td>\n",
       "      <td>0.697500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>109</td>\n",
       "      <td>0.111700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>0.413300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>111</td>\n",
       "      <td>0.123300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>112</td>\n",
       "      <td>0.219700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>113</td>\n",
       "      <td>0.111300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>114</td>\n",
       "      <td>0.063900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>115</td>\n",
       "      <td>0.314000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>116</td>\n",
       "      <td>0.188600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>117</td>\n",
       "      <td>0.303500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>118</td>\n",
       "      <td>0.042500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>119</td>\n",
       "      <td>0.136500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>0.341700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>121</td>\n",
       "      <td>0.173900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>122</td>\n",
       "      <td>0.482400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>123</td>\n",
       "      <td>0.209600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>124</td>\n",
       "      <td>0.093800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>125</td>\n",
       "      <td>0.062500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>126</td>\n",
       "      <td>0.178400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>127</td>\n",
       "      <td>0.070900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>128</td>\n",
       "      <td>0.379000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>129</td>\n",
       "      <td>0.055000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>130</td>\n",
       "      <td>0.222700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>131</td>\n",
       "      <td>0.371300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>132</td>\n",
       "      <td>0.083000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>133</td>\n",
       "      <td>0.256600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>134</td>\n",
       "      <td>0.027600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>135</td>\n",
       "      <td>0.083200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>136</td>\n",
       "      <td>0.171600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>137</td>\n",
       "      <td>0.087700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>138</td>\n",
       "      <td>0.265800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>139</td>\n",
       "      <td>0.219000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>0.029200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>141</td>\n",
       "      <td>0.109500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>142</td>\n",
       "      <td>0.132400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>143</td>\n",
       "      <td>0.017500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>144</td>\n",
       "      <td>0.107200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>145</td>\n",
       "      <td>0.014200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>146</td>\n",
       "      <td>0.028700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>147</td>\n",
       "      <td>0.086800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>148</td>\n",
       "      <td>0.172500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>149</td>\n",
       "      <td>0.024300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>0.088100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>151</td>\n",
       "      <td>0.309500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>152</td>\n",
       "      <td>0.398700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>153</td>\n",
       "      <td>0.013600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>154</td>\n",
       "      <td>0.007400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>155</td>\n",
       "      <td>0.434600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>156</td>\n",
       "      <td>0.037200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>157</td>\n",
       "      <td>0.005400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>158</td>\n",
       "      <td>0.041800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159</td>\n",
       "      <td>0.219100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>0.169400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>161</td>\n",
       "      <td>0.083100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>162</td>\n",
       "      <td>0.203100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>163</td>\n",
       "      <td>0.146300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>164</td>\n",
       "      <td>0.059300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>165</td>\n",
       "      <td>0.241900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>166</td>\n",
       "      <td>0.011200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>167</td>\n",
       "      <td>0.011900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>168</td>\n",
       "      <td>0.093100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>169</td>\n",
       "      <td>0.085000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170</td>\n",
       "      <td>0.014300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>171</td>\n",
       "      <td>0.077800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>172</td>\n",
       "      <td>0.131300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>173</td>\n",
       "      <td>0.050700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>174</td>\n",
       "      <td>0.012200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>175</td>\n",
       "      <td>0.005500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>176</td>\n",
       "      <td>0.096200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>177</td>\n",
       "      <td>0.139800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>178</td>\n",
       "      <td>0.099500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>179</td>\n",
       "      <td>0.004500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>0.007100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>181</td>\n",
       "      <td>0.070300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>182</td>\n",
       "      <td>0.165700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>183</td>\n",
       "      <td>0.040000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>184</td>\n",
       "      <td>0.005200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>185</td>\n",
       "      <td>0.092400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>186</td>\n",
       "      <td>0.057500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>187</td>\n",
       "      <td>0.017400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>188</td>\n",
       "      <td>0.070400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>189</td>\n",
       "      <td>0.040200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>190</td>\n",
       "      <td>0.007200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>191</td>\n",
       "      <td>0.017100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>192</td>\n",
       "      <td>0.002600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>193</td>\n",
       "      <td>0.006900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>194</td>\n",
       "      <td>0.029500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>195</td>\n",
       "      <td>0.000300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>196</td>\n",
       "      <td>0.028800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>197</td>\n",
       "      <td>0.018600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>198</td>\n",
       "      <td>0.020700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>199</td>\n",
       "      <td>0.021500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.024900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>201</td>\n",
       "      <td>0.000600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>202</td>\n",
       "      <td>0.035800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>203</td>\n",
       "      <td>0.011400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>204</td>\n",
       "      <td>0.001000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>205</td>\n",
       "      <td>0.008800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>206</td>\n",
       "      <td>0.112900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>207</td>\n",
       "      <td>0.014300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>208</td>\n",
       "      <td>0.033300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>209</td>\n",
       "      <td>0.002200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>210</td>\n",
       "      <td>0.111000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>211</td>\n",
       "      <td>0.039100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>212</td>\n",
       "      <td>0.087400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>213</td>\n",
       "      <td>0.027400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>214</td>\n",
       "      <td>0.003000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>215</td>\n",
       "      <td>0.186400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>216</td>\n",
       "      <td>0.005100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>217</td>\n",
       "      <td>0.147900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>218</td>\n",
       "      <td>0.317800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>219</td>\n",
       "      <td>0.051100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>0.201200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>221</td>\n",
       "      <td>0.055000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>222</td>\n",
       "      <td>0.012700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>223</td>\n",
       "      <td>0.008800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>224</td>\n",
       "      <td>0.090000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>225</td>\n",
       "      <td>0.082300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>226</td>\n",
       "      <td>0.026400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>227</td>\n",
       "      <td>0.015200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>228</td>\n",
       "      <td>0.165200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>229</td>\n",
       "      <td>0.004600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>230</td>\n",
       "      <td>0.043000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>231</td>\n",
       "      <td>0.028500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>232</td>\n",
       "      <td>0.012800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>233</td>\n",
       "      <td>0.005300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>234</td>\n",
       "      <td>0.012500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>235</td>\n",
       "      <td>0.048600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>236</td>\n",
       "      <td>0.044100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>237</td>\n",
       "      <td>0.012100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>238</td>\n",
       "      <td>0.117300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>239</td>\n",
       "      <td>0.163300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>0.002800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>241</td>\n",
       "      <td>0.007000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>242</td>\n",
       "      <td>0.031300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>243</td>\n",
       "      <td>0.101300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>244</td>\n",
       "      <td>0.092900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>245</td>\n",
       "      <td>0.015900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>246</td>\n",
       "      <td>0.000500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>247</td>\n",
       "      <td>0.002600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>248</td>\n",
       "      <td>0.004300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>249</td>\n",
       "      <td>0.075200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>0.001800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>251</td>\n",
       "      <td>0.020400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>252</td>\n",
       "      <td>0.014400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>253</td>\n",
       "      <td>0.015000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>254</td>\n",
       "      <td>0.016100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>255</td>\n",
       "      <td>0.010600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>256</td>\n",
       "      <td>0.003700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>257</td>\n",
       "      <td>0.005200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>258</td>\n",
       "      <td>0.016700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>259</td>\n",
       "      <td>0.023100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260</td>\n",
       "      <td>0.002700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>261</td>\n",
       "      <td>0.007200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>262</td>\n",
       "      <td>0.014800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>263</td>\n",
       "      <td>0.015800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>264</td>\n",
       "      <td>0.024200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>265</td>\n",
       "      <td>0.001000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>266</td>\n",
       "      <td>0.027800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>267</td>\n",
       "      <td>0.020600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>268</td>\n",
       "      <td>0.001200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>269</td>\n",
       "      <td>0.001300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>270</td>\n",
       "      <td>0.005600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>271</td>\n",
       "      <td>0.001900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>272</td>\n",
       "      <td>0.006400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>273</td>\n",
       "      <td>0.006600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>274</td>\n",
       "      <td>0.012200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>275</td>\n",
       "      <td>0.012900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>276</td>\n",
       "      <td>0.025200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>277</td>\n",
       "      <td>0.009600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>278</td>\n",
       "      <td>0.082700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>279</td>\n",
       "      <td>0.014100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>280</td>\n",
       "      <td>0.006300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>281</td>\n",
       "      <td>0.001200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>282</td>\n",
       "      <td>0.068100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>283</td>\n",
       "      <td>0.015800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>284</td>\n",
       "      <td>0.011300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>285</td>\n",
       "      <td>0.012700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>286</td>\n",
       "      <td>0.000300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>287</td>\n",
       "      <td>0.012600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>288</td>\n",
       "      <td>0.006100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>289</td>\n",
       "      <td>0.654900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>290</td>\n",
       "      <td>0.001100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>291</td>\n",
       "      <td>0.002800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>292</td>\n",
       "      <td>0.003600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>293</td>\n",
       "      <td>0.015700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>294</td>\n",
       "      <td>0.002300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>295</td>\n",
       "      <td>0.001400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>296</td>\n",
       "      <td>0.005000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297</td>\n",
       "      <td>0.003500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>298</td>\n",
       "      <td>0.001900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>299</td>\n",
       "      <td>0.038600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.003400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>301</td>\n",
       "      <td>0.004100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>302</td>\n",
       "      <td>0.010900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>303</td>\n",
       "      <td>0.000500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>304</td>\n",
       "      <td>0.012800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>305</td>\n",
       "      <td>0.004400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /mnt/workspace/.cache/modelscope/hub/Qwen/Qwen2-7B-Instruct - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mswanlab\u001b[0m: Step 305 on key train/epoch already exists, ignored.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=305, training_loss=0.24218164794690922, metrics={'train_runtime': 1859.3728, 'train_samples_per_second': 2.635, 'train_steps_per_second': 0.164, 'total_flos': 7.969547737228493e+16, 'train_loss': 0.24218164794690922, 'epoch': 4.979591836734694})"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🌟 Run `\u001b[1mswanlab watch -l /mnt/workspace/swanlog\u001b[0m` to view SwanLab Experiment Dashboard locally\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🏠 View project at \u001b[34m\u001b[4mhttps://swanlab.cn/@zs-train/Qwen2-fintune\u001b[0m\u001b[0m\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://swanlab.cn/@zs-train/Qwen2-fintune/runs/1ak1fest0vzqqzvr2jab7\u001b[0m\u001b[0m\n",
      "                                                                                                    \r"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=tokenized_id,\n",
    "    data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),\n",
    "    callbacks=[swanlab_callback]\n",
    ")\n",
    "trainer.train()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "23ee76fd-1823-46cf-8a65-7ec5c6ae826a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-01T07:33:26.137738Z",
     "iopub.status.busy": "2024-08-01T07:33:26.137417Z",
     "iopub.status.idle": "2024-08-01T07:33:27.750700Z",
     "shell.execute_reply": "2024-08-01T07:33:27.750352Z",
     "shell.execute_reply.started": "2024-08-01T07:33:26.137717Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "swanlab.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0845f28-3e8b-4829-8690-593cd2b08ce5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e2647c98-26d7-4cfd-9a25-6380d8c161ed",
   "metadata": {},
   "source": [
    "# 训练完推理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "25b0849c-1d63-429a-8517-1a0e609f03c1",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T07:34:51.805820Z",
     "iopub.status.busy": "2024-08-01T07:34:51.805513Z",
     "iopub.status.idle": "2024-08-01T07:35:01.179645Z",
     "shell.execute_reply": "2024-08-01T07:35:01.179159Z",
     "shell.execute_reply.started": "2024-08-01T07:34:51.805799Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Loading checkpoint shards: 100%|██████████| 4/4 [00:03<00:00,  1.30it/s]\n"
     ]
    }
   ],
   "source": [
    "from modelscope import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "from peft import PeftModel\n",
    "\n",
    "mode_path = 'Qwen/Qwen2-7B-Instruct'\n",
    "lora_path = './Qwen/Qwen2_pinpai_1000/checkpoint-300' # 这里改称你的 lora 输出对应 checkpoint 地址\n",
    "\n",
    "# 加载tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(mode_path, trust_remote_code=True)\n",
    "\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(mode_path, device_map=\"auto\", torch_dtype=torch.bfloat16, trust_remote_code=True).eval()\n",
    "\n",
    "# 加载lora权重\n",
    "model = PeftModel.from_pretrained(model, model_id=lora_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6dca61f-dfe9-4429-8e96-31a552579581",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 加载完预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "85893996-04aa-4f6e-9225-a20bd951768b",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-08-01T07:35:25.573880Z",
     "iopub.status.busy": "2024-08-01T07:35:25.573550Z",
     "iopub.status.idle": "2024-08-01T07:35:39.058740Z",
     "shell.execute_reply": "2024-08-01T07:35:39.058383Z",
     "shell.execute_reply.started": "2024-08-01T07:35:25.573859Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[“粉底液-zati\"]\n",
      "[“不是商品文章”]\n",
      "[“内饰-威飒\"]\n",
      "[“美瞳-aisei”]\n",
      "[“电视-海信\"]\n",
      "[“水牛奶-摩拉尤品\"]\n",
      "[“不是商品文章”]\n",
      "[“奶茶-霸王茶姬\"]\n",
      "[“不是商品文章”]\n",
      "[“票-dearjane”]\n",
      "[“饮料-娃哈哈\"]\n",
      "[“衣服-stussy\"]\n",
      "[“二手车-福特\"]\n",
      "[“套装-安妮家\"]\n",
      "[“不是商品文章”]\n",
      "[“珠宝-六福\"]\n",
      "[“不是商品文章”]\n",
      "[“不是商品文章”]\n",
      "[“手机-苹果\"]\n",
      "[“相机-富士\"]\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: Tracking run with swanlab version 0.3.9                                   \n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: Run data will be saved locally in \u001b[35m\u001b[1m/mnt/workspace/swanlog/run-20240801_153536-9fabef00\u001b[0m\u001b[0m\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 👋 Hi \u001b[1m\u001b[39mzs-train\u001b[0m\u001b[0m, welcome to swanlab!\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: Syncing run \u001b[33mQwen2-7B-Instruct-pinpai_1000_Aug01_15-35-36\u001b[0m to the cloud\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🌟 Run `\u001b[1mswanlab watch -l /mnt/workspace/swanlog\u001b[0m` to view SwanLab Experiment Dashboard locally\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🏠 View project at \u001b[34m\u001b[4mhttps://swanlab.cn/@zs-train/Qwen2-fintune\u001b[0m\u001b[0m\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://swanlab.cn/@zs-train/Qwen2-fintune/runs/7c0e6h5409pqwrisvv02e\u001b[0m\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<!DOCTYPE html>\n",
       "<html lang=\"en\">\n",
       "<head>\n",
       "    <meta charset=\"UTF-8\">\n",
       "    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n",
       "    <title>Show Iframe</title>\n",
       "    \n",
       "        <script>\n",
       "            function showIframe() {\n",
       "                var iframeHtml = '<iframe src=\"https://swanlab.cn/@zs-train/Qwen2-fintune/runs/7c0e6h5409pqwrisvv02e\" width=100% height=\"600\" frameborder=\"no\"></iframe>';\n",
       "                document.getElementById('iframeContainer').innerHTML = iframeHtml;\n",
       "            }\n",
       "        </script>\n",
       "        \n",
       "</head>\n",
       "<body>\n",
       "    <style>\n",
       "        .interactive-button {\n",
       "            display: flex;\n",
       "            align-items: center;\n",
       "            height: 36px;\n",
       "            border: 0px;\n",
       "            background-color: #2c8f63;\n",
       "            color: white;\n",
       "            padding: 10px 20px;\n",
       "            transition: background-color 0.3s, transform 0.2s;\n",
       "        }\n",
       "\n",
       "        .interactive-button:hover {\n",
       "            background-color: #5cab87;\n",
       "            cursor: pointer;\n",
       "        }\n",
       "\n",
       "        .interactive-button:active {\n",
       "            background-color: #217952;\n",
       "            transform: scale(0.96);\n",
       "        }\n",
       "    </style>\n",
       "    <br>\n",
       "    <button onclick=\"showIframe()\" class=\"interactive-button\">\n",
       "        <svg style=\"height: 16px; margin-right: 8px;\" viewBox=\"0 0 46 46\" fill=\"none\">\n",
       "            <path\n",
       "                d=\"M10.8439 21.1974C10.6414 21.2854 10.4477 21.3925 10.2655 21.5173L10.2069 21.5652C10.1839 21.58 10.1625 21.5969 10.1429 21.6159C6.29135 24.6118 4.22831 29.4416 5.32646 34.282C5.94656 37.0577 7.50461 39.5348 9.73801 41.2958C11.9714 43.0568 14.7436 43.994 17.5874 43.9495H18.0219C19.8864 43.8697 21.7087 43.3694 23.3526 42.486C24.9964 41.6026 26.4193 40.3589 27.5147 38.848C28.61 37.3371 29.3496 35.598 29.678 33.761C30.0065 31.9239 29.9153 30.0363 29.4112 28.2395C28.9181 26.4723 27.8919 24.8437 26.9937 23.2551C25.4158 20.4653 23.8343 17.6764 22.2492 14.8884C21.7801 14.0647 21.3057 13.2465 20.8419 12.4228C20.2315 11.3353 19.2746 10.1519 19.224 8.86183C19.1733 7.57176 20.2235 6.32701 21.5082 6.07912C23.9284 5.61801 25.0639 8.24078 25.0693 8.23812C25.363 8.94035 25.9123 9.50489 26.6063 9.81764C27.3002 10.1304 28.087 10.168 28.8077 9.92298C29.5283 9.67791 30.1291 9.1684 30.4885 8.49743C30.8479 7.82646 30.9392 7.04405 30.7439 6.30835C30.1514 4.37314 28.9133 2.69953 27.2363 1.56656C25.7615 0.511704 23.9847 -0.0372109 22.1719 0.00195984C20.9049 0.00893199 19.6532 0.27989 18.4967 0.797557C17.3402 1.31522 16.3043 2.06823 15.4551 3.00856C14.49 4.08707 13.7984 5.38193 13.4389 6.78385C13.0794 8.18576 13.0624 9.6536 13.3894 11.0635C13.52 11.593 13.6984 12.1095 13.9225 12.6067C14.5595 14.0514 15.4951 15.3681 16.284 16.7355C17.2525 18.4147 18.2209 20.0948 19.1893 21.7758C20.1578 23.4568 21.1351 25.1449 22.1213 26.8401C22.9209 28.2421 23.7925 29.4682 23.8805 31.1528C23.9175 32.0513 23.7682 32.9479 23.4419 33.7859C23.1156 34.6239 22.6194 35.3854 21.9845 36.0223C21.3496 36.6592 20.5897 37.1578 19.7527 37.4868C18.9157 37.8157 18.0196 37.9678 17.121 37.9336C14.0024 37.7923 11.6488 35.4814 11.1744 32.4588C10.58 28.6419 13.552 26.5469 13.552 26.5469C14.1782 26.1785 14.6497 25.5955 14.8791 24.906C15.1084 24.2166 15.0801 23.4673 14.7993 22.7971C14.5186 22.127 14.0044 21.5813 13.3521 21.2611C12.6998 20.941 11.9536 20.8682 11.2517 21.0561C11.1174 21.0939 10.9856 21.1402 10.8572 21.1947\"\n",
       "                fill=\"white\"\n",
       "            />\n",
       "            <path\n",
       "                d=\"M42.8101 31.5968C42.8109 30.5198 42.7218 29.4445 42.5435 28.3823C42.2663 26.7069 41.7464 25.0808 41.0002 23.5552C40.5524 22.6463 39.9874 21.7374 39.1024 21.2417C38.6593 20.9919 38.1589 20.8617 37.6502 20.8639C37.1416 20.8661 36.6423 21.0006 36.2013 21.2541C35.7604 21.5077 35.393 21.8716 35.1352 22.3101C34.8775 22.7485 34.7382 23.2466 34.7312 23.7552C34.7072 24.8773 35.3149 25.8875 35.768 26.9217C36.5212 28.6453 36.8623 30.5208 36.7642 32.3993C36.6661 34.2777 36.1315 36.1075 35.2029 37.7433C35.146 37.8404 35.0952 37.941 35.051 38.0445C34.8623 38.4842 34.7635 38.9573 34.7605 39.4358C34.7802 40.1222 35.0356 40.7808 35.4835 41.3011C35.9315 41.8214 36.5449 42.1717 37.2207 42.2932C38.8759 42.589 40.1899 41.347 40.8856 39.9609C42.1643 37.3589 42.823 34.4961 42.8101 31.5968Z\"\n",
       "                fill=\"white\"\n",
       "            />\n",
       "            <path\n",
       "                d=\"M28.2309 11.8938C28.1761 11.9043 28.1218 11.9176 28.0683 11.9338C27.9593 11.9642 27.8611 12.0249 27.7851 12.1088C27.7091 12.1928 27.6584 12.2965 27.6389 12.408C27.6193 12.5195 27.6318 12.6343 27.6748 12.7391C27.7178 12.8438 27.7895 12.9343 27.8818 12.9999C29.2375 14.0252 30.3809 15.3043 31.2482 16.7662C31.4838 17.1677 31.6888 17.5865 31.8612 18.0189C32.0052 18.3921 32.1971 18.8799 32.6822 18.8532C33.0607 18.8346 33.2153 18.512 33.3192 18.1895C33.8137 16.5125 33.9678 14.7534 33.7723 13.0159C33.6331 12.0693 33.4155 11.1359 33.122 10.2252C33.0775 10.0047 32.9744 9.80029 32.8235 9.6335C32.7273 9.54627 32.6054 9.49262 32.4761 9.4806C32.3468 9.46859 32.2171 9.49886 32.1065 9.56687C32.0016 9.65188 31.9115 9.75365 31.8399 9.86806C31.3956 10.4658 30.825 10.9581 30.1687 11.3101C29.8377 11.4861 29.4893 11.6272 29.1292 11.7312C28.828 11.8192 28.5215 11.8325 28.2309 11.8938Z\"\n",
       "                fill=\"white\"\n",
       "            />\n",
       "        </svg>\n",
       "        Display SwanLab Board\n",
       "    </button>\n",
       "    <br>\n",
       "    <div id=\"iframeContainer\"></div>\n",
       "</body>\n",
       "</html>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🌟 Run `\u001b[1mswanlab watch -l /mnt/workspace/swanlog\u001b[0m` to view SwanLab Experiment Dashboard locally\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🏠 View project at \u001b[34m\u001b[4mhttps://swanlab.cn/@zs-train/Qwen2-fintune\u001b[0m\u001b[0m\n",
      "\u001b[1m\u001b[34mswanlab\u001b[0m\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://swanlab.cn/@zs-train/Qwen2-fintune/runs/7c0e6h5409pqwrisvv02e\u001b[0m\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                                    \r"
     ]
    }
   ],
   "source": [
    "def predict(messages, model, tokenizer):\n",
    "    device = \"cuda\"\n",
    "    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
    "    model_inputs = tokenizer([text], return_tensors=\"pt\").to(device)\n",
    "    generated_ids = model.generate(model_inputs.input_ids, max_new_tokens=512)\n",
    "    generated_ids = [\n",
    "        output_ids[len(input_ids) :]\n",
    "        for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)\n",
    "    ]\n",
    "\n",
    "    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]\n",
    "    print(response)\n",
    "\n",
    "    return response\n",
    "\n",
    "\n",
    "test_text_list = []\n",
    "for index, row in test_df.iterrows():\n",
    "    instruction = row[\"instruction\"]\n",
    "    input_value = row[\"input\"]\n",
    "\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": f\"{instruction}\"},\n",
    "        {\"role\": \"user\", \"content\": f\"{input_value}\"},\n",
    "    ]\n",
    "\n",
    "    response = predict(messages, model, tokenizer)\n",
    "    messages.append({\"role\": \"assistant\", \"content\": f\"{response}\"})\n",
    "    result_text = f\"{messages[0]}\\n\\n{messages[1]}\\n\\n{messages[2]}\"\n",
    "    test_text_list.append(swanlab.Text(result_text, caption=response))\n",
    "swanlab.init(\n",
    "    project=\"Qwen2-fintune\",\n",
    "    experiment_name=\"Qwen2-7B-Instruct-pinpai_1000\",\n",
    "    description=\"使用通义千问Qwen2-7B-Instruct模型在pinpai1000微调。\",\n",
    "    config={\n",
    "        \"model\": \"Qwen/Qwen2-7B-Instruct\",\n",
    "        \"dataset\": \"xhspinpai.json\",\n",
    "    })\n",
    "swanlab.log({\"Prediction\": test_text_list})\n",
    "swanlab.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6072e92b-3cea-4ca1-8350-d8acefa74c63",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "683cd3bd-9500-4f32-a680-bbf6306bdce7",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-07-11T09:44:31.001803Z",
     "iopub.status.busy": "2024-07-11T09:44:31.001410Z",
     "iopub.status.idle": "2024-07-11T09:44:35.540924Z",
     "shell.execute_reply": "2024-07-11T09:44:35.540324Z",
     "shell.execute_reply.started": "2024-07-11T09:44:31.001777Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "人物相关\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "prompt = '''你叫什么名字'''\n",
    "inputs = tokenizer.apply_chat_template([{\"role\": \"user\", \"content\": \"你是一个文本分类领域的专家，你会接收到一段文本和几个潜在的分类选项，请输出文本内容的正确类型\"},{\"role\": \"user\", \"content\": prompt}],\n",
    "                                       add_generation_prompt=True,\n",
    "                                       tokenize=True,\n",
    "                                       return_tensors=\"pt\",\n",
    "                                       return_dict=True\n",
    "                                       ).to(device)\n",
    "\n",
    "\n",
    "gen_kwargs = {\"max_length\": 2500, \"do_sample\": True, \"top_k\": 1}\n",
    "with torch.no_grad():\n",
    "    outputs = model.generate(**inputs, **gen_kwargs)\n",
    "    outputs = outputs[:, inputs['input_ids'].shape[1]:]\n",
    "    print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1126e266-978e-40d3-ac76-a1408d8a5084",
   "metadata": {},
   "source": [
    "## webui"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dfbae6c5-fd32-48b0-a817-cb6db41f354f",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-07-11T09:31:40.789312Z",
     "iopub.status.busy": "2024-07-11T09:31:40.788943Z",
     "iopub.status.idle": "2024-07-11T09:31:42.503567Z",
     "shell.execute_reply": "2024-07-11T09:31:42.502999Z",
     "shell.execute_reply.started": "2024-07-11T09:31:40.789288Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import gradio as gr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "de909452-5f64-4360-a987-3f10c18d8a9c",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-07-11T09:45:14.263159Z",
     "iopub.status.busy": "2024-07-11T09:45:14.262558Z",
     "iopub.status.idle": "2024-07-11T09:45:16.495050Z",
     "shell.execute_reply": "2024-07-11T09:45:16.494452Z",
     "shell.execute_reply.started": "2024-07-11T09:45:14.263131Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7862\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"http://127.0.0.1:7862/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.10/site-packages/gradio/queueing.py\", line 541, in process_events\n",
      "    response = await route_utils.call_process_api(\n",
      "  File \"/usr/local/lib/python3.10/site-packages/gradio/route_utils.py\", line 276, in call_process_api\n",
      "    output = await app.get_blocks().process_api(\n",
      "  File \"/usr/local/lib/python3.10/site-packages/gradio/blocks.py\", line 1928, in process_api\n",
      "    result = await self.call_function(\n",
      "  File \"/usr/local/lib/python3.10/site-packages/gradio/blocks.py\", line 1512, in call_function\n",
      "    prediction = await fn(*processed_input)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/gradio/utils.py\", line 800, in async_wrapper\n",
      "    response = await f(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/gradio/chat_interface.py\", line 554, in _submit_fn\n",
      "    response = await anyio.to_thread.run_sync(\n",
      "  File \"/usr/local/lib/python3.10/site-packages/anyio/to_thread.py\", line 56, in run_sync\n",
      "    return await get_async_backend().run_sync_in_worker_thread(\n",
      "  File \"/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py\", line 2177, in run_sync_in_worker_thread\n",
      "    return await future\n",
      "  File \"/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py\", line 859, in run\n",
      "    result = context.run(func, *args)\n",
      "  File \"/tmp/ipykernel_4287/3320625168.py\", line 12, in continue_dialogue\n",
      "    outputs = model.generate(**inputs, **gen_kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/peft/peft_model.py\", line 1491, in generate\n",
      "    outputs = self.base_model.generate(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/torch/utils/_contextlib.py\", line 115, in decorate_context\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/transformers/generation/utils.py\", line 1758, in generate\n",
      "    result = self._sample(\n",
      "  File \"/usr/local/lib/python3.10/site-packages/transformers/generation/utils.py\", line 2397, in _sample\n",
      "    outputs = self(\n",
      "  File \"/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
      "    return self._call_impl(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/accelerate/hooks.py\", line 166, in new_forward\n",
      "    output = module._old_forward(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py\", line 1162, in forward\n",
      "    logits = self.lm_head(hidden_states)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
      "    return self._call_impl(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/accelerate/hooks.py\", line 161, in new_forward\n",
      "    args, kwargs = module._hf_hook.pre_forward(module, *args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/site-packages/accelerate/hooks.py\", line 347, in pre_forward\n",
      "    set_module_tensor_to_device(\n",
      "  File \"/usr/local/lib/python3.10/site-packages/accelerate/utils/modeling.py\", line 400, in set_module_tensor_to_device\n",
      "    new_value = value.to(device)\n",
      "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.02 GiB. GPU \n"
     ]
    }
   ],
   "source": [
    "def continue_dialogue(user_input, reset=False):\n",
    "    expert_prompt = \"你是一个文本分类领域的专家，你会接收到一段文本并输出文本内容的正确类型，以下为给出的内容：\"\n",
    "    inputs = tokenizer.apply_chat_template([{\"role\": \"user\", \"content\":expert_prompt+\"\\n\"  + user_input}],\n",
    "                                           add_generation_prompt=True,\n",
    "                                           tokenize=True,\n",
    "                                           return_tensors=\"pt\",\n",
    "                                           return_dict=True)\n",
    "    inputs = {k: v.to(device) for k, v in inputs.items()}\n",
    "    # 生成响应\n",
    "    gen_kwargs = {\"max_length\": 2500, \"do_sample\": True, \"top_k\": 2}\n",
    "    with torch.no_grad():\n",
    "        outputs = model.generate(**inputs, **gen_kwargs)\n",
    "        outputs = outputs[:, inputs['input_ids'].shape[1]:]\n",
    "    response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
    "    return response\n",
    "\n",
    "title = \"Transformer Chatbot\"\n",
    "description = \"Enter your message and receive a response from the transformer-based language model.\"\n",
    "\n",
    "iface = gr.ChatInterface(\n",
    "    fn=continue_dialogue,\n",
    "    title=title,\n",
    "    description=description,\n",
    "    examples=[[\"这里可以使用你微调的数据代替\"],\n",
    "              [\"你叫什么名字?\"]]\n",
    ")\n",
    "iface.launch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "3e5c8f14-1669-45d0-8f68-a623449302e4",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-07-11T08:51:45.916054Z",
     "iopub.status.busy": "2024-07-11T08:51:45.915028Z",
     "iopub.status.idle": "2024-07-11T08:51:48.158173Z",
     "shell.execute_reply": "2024-07-11T08:51:48.157653Z",
     "shell.execute_reply.started": "2024-07-11T08:51:45.916022Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7895\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"http://127.0.0.1:7895/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dialogue_history = []\n",
    "def continue_dialogue(user_input, reset=False):\n",
    "    global dialogue_history\n",
    "    if reset:\n",
    "        dialogue_history = []  # 重置对话历史\n",
    "    dialogue_history.append({\"role\": \"user\", \"content\": \"你是一个文本分类领域的专家，你会接收到一段文本并输出文本内容的正确类型，以下为给出的内容：\"})\n",
    "    dialogue_history.append({\"role\": \"user\", \"content\": user_input})\n",
    "    # 生成当前对话的输入\n",
    "    inputs = tokenizer.apply_chat_template(dialogue_history,\n",
    "                                           add_generation_prompt=True,\n",
    "                                           tokenize=True,\n",
    "                                           return_tensors=\"pt\",\n",
    "                                           return_dict=True)\n",
    "    inputs = {k: v.to(device) for k, v in inputs.items()}\n",
    "    # 生成响应\n",
    "    gen_kwargs = {\"max_length\": 2500, \"do_sample\": True, \"top_k\": 3}\n",
    "    with torch.no_grad():\n",
    "        outputs = model.generate(**inputs, **gen_kwargs)\n",
    "        outputs = outputs[:, inputs['input_ids'].shape[1]:]\n",
    "        response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
    "    dialogue_history.append({\"role\": \"model\", \"content\": response})\n",
    "    return response\n",
    "\n",
    "\n",
    "# def reset_dialogue():\n",
    "    # continue_dialogue(\"\", reset=True)\n",
    "    # return \"对话历史已重置！\"\n",
    "\n",
    "\n",
    "title = \"Transformer Chatbot\"\n",
    "description = \"Enter your message and receive a response from the transformer-based language model.\"\n",
    "iface = gr.ChatInterface(\n",
    "    fn=continue_dialogue,\n",
    "    title=title,\n",
    "    description=description,\n",
    "    examples=[[\"这里可以使用你微调的数据代替\"],\n",
    "              [\"你叫什么名字?\"]]\n",
    ")\n",
    "iface.launch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a863902-6cbd-438e-9780-464bf6e2e116",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0c40ddd-7cb8-4780-a097-2a975793a496",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e50432d-62c7-4895-8e86-cd23ad6110f1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ee4f143-e78f-426e-9808-3c2b91952b5d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f17d1c39-7941-4d2d-b882-8a51a7c71520",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f59f2693-3034-4343-b93b-42738b0a09af",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff460793-816d-445f-bfac-89395f543444",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "503ea081-fd9a-4ebb-a8f1-62ba1f407154",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}