gemma-7b-it-inference-roleplay.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":30683,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U bitsandbytes\n!pip install -U transformers\n!pip install -U accelerate\n!pip install -U peft","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-15T18:41:21.937707Z","iopub.execute_input":"2024-04-15T18:41:21.938027Z","iopub.status.idle":"2024-04-15T18:42:18.587245Z","shell.execute_reply.started":"2024-04-15T18:41:21.937998Z","shell.execute_reply":"2024-04-15T18:42:18.586152Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting bitsandbytes\n  Downloading bitsandbytes-0.43.1-py3-none-manylinux_2_24_x86_64.whl.metadata (2.2 kB)\nRequirement already satisfied: torch in /opt/conda/lib/python3.10/site-packages (from bitsandbytes) (2.1.2)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from bitsandbytes) (1.26.4)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (3.13.1)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (4.9.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (3.2.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (3.1.2)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (2024.2.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch->bitsandbytes) (2.1.3)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch->bitsandbytes) (1.3.0)\nDownloading bitsandbytes-0.43.1-py3-none-manylinux_2_24_x86_64.whl (119.8 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.8/119.8 MB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hInstalling collected packages: bitsandbytes\nSuccessfully installed bitsandbytes-0.43.1\nRequirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.39.3)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.13.1)\nRequirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.22.2)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.12.25)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers) (2.31.0)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.15.2)\nRequirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.4.2)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers) (4.66.1)\nRequirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2024.2.0)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.9.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.1.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2024.2.2)\nRequirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (0.28.0)\nCollecting accelerate\n  Downloading accelerate-0.29.2-py3-none-any.whl.metadata (18 kB)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate) (6.0.1)\nRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (2.1.2)\nRequirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.22.2)\nRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.4.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate) (3.1.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.13.1)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (4.9.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.2.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (2024.2.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (2.31.0)\nRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (4.66.1)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\nDownloading accelerate-0.29.2-py3-none-any.whl (297 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m297.4/297.4 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hInstalling collected packages: accelerate\n  Attempting uninstall: accelerate\n    Found existing installation: accelerate 0.28.0\n    Uninstalling accelerate-0.28.0:\n      Successfully uninstalled accelerate-0.28.0\nSuccessfully installed accelerate-0.29.2\nCollecting peft\n  Downloading peft-0.10.0-py3-none-any.whl.metadata (13 kB)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from peft) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from peft) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from peft) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from peft) (6.0.1)\nRequirement already satisfied: torch>=1.13.0 in /opt/conda/lib/python3.10/site-packages (from peft) (2.1.2)\nRequirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (from peft) (4.39.3)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from peft) (4.66.1)\nRequirement already satisfied: accelerate>=0.21.0 in /opt/conda/lib/python3.10/site-packages (from peft) (0.29.2)\nRequirement already satisfied: safetensors in /opt/conda/lib/python3.10/site-packages (from peft) (0.4.2)\nRequirement already satisfied: huggingface-hub>=0.17.0 in /opt/conda/lib/python3.10/site-packages (from peft) (0.22.2)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.17.0->peft) (3.13.1)\nRequirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.17.0->peft) (2024.2.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.17.0->peft) (2.31.0)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.17.0->peft) (4.9.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->peft) (3.1.1)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->peft) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->peft) (3.2.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->peft) (3.1.2)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers->peft) (2023.12.25)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /opt/conda/lib/python3.10/site-packages (from transformers->peft) (0.15.2)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.13.0->peft) (2.1.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub>=0.17.0->peft) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub>=0.17.0->peft) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub>=0.17.0->peft) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub>=0.17.0->peft) (2024.2.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.13.0->peft) (1.3.0)\nDownloading peft-0.10.0-py3-none-any.whl (199 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.1/199.1 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n\u001b[?25hInstalling collected packages: peft\nSuccessfully installed peft-0.10.0\n","output_type":"stream"}]},{"cell_type":"code","source":"!pip install huggingface_hub","metadata":{"execution":{"iopub.status.busy":"2024-04-15T18:42:18.589521Z","iopub.execute_input":"2024-04-15T18:42:18.589839Z","iopub.status.idle":"2024-04-15T18:42:31.163440Z","shell.execute_reply.started":"2024-04-15T18:42:18.589810Z","shell.execute_reply":"2024-04-15T18:42:31.162326Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Requirement already satisfied: huggingface_hub in /opt/conda/lib/python3.10/site-packages (0.22.2)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (3.13.1)\nRequirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2024.2.0)\nRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (6.0.1)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2.31.0)\nRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.66.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.9.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.9->huggingface_hub) (3.1.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (2024.2.2)\n","output_type":"stream"}]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nHF_TOKEN_READ = user_secrets.get_secret(\"HF_TOKEN\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T18:48:05.808796Z","iopub.execute_input":"2024-04-15T18:48:05.809697Z","iopub.status.idle":"2024-04-15T18:48:05.987143Z","shell.execute_reply.started":"2024-04-15T18:48:05.809663Z","shell.execute_reply":"2024-04-15T18:48:05.986427Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"!huggingface-cli login --token $HF_TOKEN_READ","metadata":{"execution":{"iopub.status.busy":"2024-04-15T18:48:08.860116Z","iopub.execute_input":"2024-04-15T18:48:08.860939Z","iopub.status.idle":"2024-04-15T18:48:10.440853Z","shell.execute_reply.started":"2024-04-15T18:48:08.860893Z","shell.execute_reply":"2024-04-15T18:48:10.439709Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Token has not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\nToken is valid (permission: read).\nYour token has been saved to /root/.cache/huggingface/token\nLogin successful\n","output_type":"stream"}]},{"cell_type":"code","source":"base_model = \"google/gemma-7b-it\"\nnew_model = \"APaul1/gemma-7b-it-v2-role-play\"","metadata":{"execution":{"iopub.status.busy":"2024-04-15T18:49:51.278807Z","iopub.execute_input":"2024-04-15T18:49:51.279536Z","iopub.status.idle":"2024-04-15T18:49:51.283866Z","shell.execute_reply.started":"2024-04-15T18:49:51.279503Z","shell.execute_reply":"2024-04-15T18:49:51.282898Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline\nfrom peft import PeftModel\nimport torch","metadata":{"execution":{"iopub.status.busy":"2024-04-15T18:49:14.875840Z","iopub.execute_input":"2024-04-15T18:49:14.876725Z","iopub.status.idle":"2024-04-15T18:49:35.448938Z","shell.execute_reply.started":"2024-04-15T18:49:14.876692Z","shell.execute_reply":"2024-04-15T18:49:35.447878Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stderr","text":"2024-04-15 18:49:22.850961: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n2024-04-15 18:49:22.851060: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n2024-04-15 18:49:23.019837: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n","output_type":"stream"}]},{"cell_type":"code","source":"base_model_reload = AutoModelForCausalLM.from_pretrained(\n        base_model,\n        return_dict=True,\n        low_cpu_mem_usage=True,\n        torch_dtype=torch.float16,\n        device_map=\"auto\",\n        trust_remote_code=True,\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T18:49:53.624671Z","iopub.execute_input":"2024-04-15T18:49:53.625165Z","iopub.status.idle":"2024-04-15T18:52:12.831349Z","shell.execute_reply.started":"2024-04-15T18:49:53.625134Z","shell.execute_reply":"2024-04-15T18:52:12.830386Z"},"trusted":true},"execution_count":11,"outputs":[{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/694 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a42eaaff78a148b683e5bae3c5fbde01"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors.index.json:   0%|          | 0.00/20.9k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cae2166e92914cd5804d1e3eebf7618d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading shards:   0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a3e7e3b2a46542bfb746645d74f60bcd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model-00001-of-00004.safetensors:   0%|          | 0.00/5.00G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"56fb5c48f1bc453cb8f242c1fea9f5db"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model-00002-of-00004.safetensors:   0%|          | 0.00/4.98G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ec64e800ffbd437f988161ae97dd344e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model-00003-of-00004.safetensors:   0%|          | 0.00/4.98G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ea73136fd1fe4d86920f1de346db009e"}},"metadata":{}},{"name":"stderr","text":"Gemma's activation function should be approximate GeLU and not exact GeLU.\nChanging the activation function to `gelu_pytorch_tanh`.if you want to use the legacy `gelu`, edit the `model.config` to set `hidden_activation=gelu`   instead of `hidden_act`. See https://github.com/huggingface/transformers/pull/29402 for more details.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"96f5bf10080e4f53a1661781dfb391c1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/137 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"772ac586558349df8c555f564bf5d7c7"}},"metadata":{}}]},{"cell_type":"code","source":"model = PeftModel.from_pretrained(base_model_reload, new_model)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T18:52:12.833331Z","iopub.execute_input":"2024-04-15T18:52:12.834081Z","iopub.status.idle":"2024-04-15T18:52:33.351091Z","shell.execute_reply.started":"2024-04-15T18:52:12.834044Z","shell.execute_reply":"2024-04-15T18:52:33.350248Z"},"trusted":true},"execution_count":12,"outputs":[{"output_type":"display_data","data":{"text/plain":"adapter_config.json:   0%|          | 0.00/724 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e5ab8c13391d4c54b0d4191893c70ea7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"adapter_model.safetensors:   0%|          | 0.00/800M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2ee8b8e05c7f43e18397db7e2b7104dd"}},"metadata":{}}]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(base_model, trust_remote_code=True)\ntokenizer.pad_token = tokenizer.eos_token\ntokenizer.padding_side = \"right\"","metadata":{"execution":{"iopub.status.busy":"2024-04-15T18:52:33.352643Z","iopub.execute_input":"2024-04-15T18:52:33.353393Z","iopub.status.idle":"2024-04-15T18:52:35.326988Z","shell.execute_reply.started":"2024-04-15T18:52:33.353356Z","shell.execute_reply":"2024-04-15T18:52:35.326092Z"},"trusted":true},"execution_count":13,"outputs":[{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/2.16k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2442e090c5ba491295dff50565144520"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.model:   0%|          | 0.00/4.24M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2029c39b5c1648568b37fed8d54e8f4b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json:   0%|          | 0.00/17.5M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f160b55ec8d6483691bc44a42d68d40d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/888 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"47f47ad197f44438bd4bde620db14730"}},"metadata":{}}]},{"cell_type":"code","source":"prompt = '''<|system|> Alan Watts's colorful journey explored the depths of philosophy and religion, weaving together Eastern wisdom and Western insights.\n<|user|> What does the self actually amount to? \n<|assistant|>'''\n    \ninputs = tokenizer(prompt, return_tensors='pt', padding=True, truncation=True).to(\"cuda\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\noutputs = model.generate(**inputs, max_length=500, num_return_sequences=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = tokenizer.decode(outputs[0], skip_special_tokens=True)\n\nprint(text)","metadata":{},"execution_count":null,"outputs":[]}]}