From 7197c1db41efb8694217fb5e8091fa45252babc2 Mon Sep 17 00:00:00 2001 From: Roman Bredehoft Date: Tue, 6 Aug 2024 17:01:45 +0200 Subject: [PATCH] chore: add lm_head --- script/make_utils/run_use_case_examples.sh | 2 +- .../lora_finetune/gpt2_finetune_hybrid.ipynb | 161 +++++++++--------- 2 files changed, 85 insertions(+), 78 deletions(-) diff --git a/script/make_utils/run_use_case_examples.sh b/script/make_utils/run_use_case_examples.sh index 402e202c73..630442ff9c 100755 --- a/script/make_utils/run_use_case_examples.sh +++ b/script/make_utils/run_use_case_examples.sh @@ -41,7 +41,7 @@ install_requirements() { local example_dir=$1 if [ -f "${example_dir}/requirements.txt" ]; then pushd "$example_dir" - if pip install -r requirements.txt; then + if pip install -r requirements.txt --extra-index-url https://pypi.zama.ai/cpu; then echo "Requirements installed successfully." else echo "Failed to install requirements." diff --git a/use_case_examples/lora_finetune/gpt2_finetune_hybrid.ipynb b/use_case_examples/lora_finetune/gpt2_finetune_hybrid.ipynb index 1dec89ff0d..1a23107677 100644 --- a/use_case_examples/lora_finetune/gpt2_finetune_hybrid.ipynb +++ b/use_case_examples/lora_finetune/gpt2_finetune_hybrid.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 100, "metadata": {}, "outputs": [], "source": [ @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 101, "metadata": {}, "outputs": [], "source": [ @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 102, "metadata": {}, "outputs": [], "source": [ @@ -83,7 +83,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 103, "metadata": {}, "outputs": [ { @@ -103,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 104, "metadata": {}, "outputs": [], "source": [ @@ -120,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 105, "metadata": {}, "outputs": [], "source": [ @@ -148,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 106, "metadata": {}, "outputs": [], "source": [ @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 107, "metadata": {}, "outputs": [], "source": [ @@ -175,13 +175,13 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 108, "metadata": {}, "outputs": [], "source": [ "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)\n", "\n", - "EPOCHS = 100\n", + "EPOCHS = 2\n", "PER_DEVICE_TRAIN_BATCH_SIZE = 4\n", "\n", "training_args = TrainingArguments(\n", @@ -204,7 +204,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 109, "metadata": {}, "outputs": [], "source": [ @@ -229,7 +229,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 110, "metadata": {}, "outputs": [], "source": [ @@ -240,7 +240,9 @@ " # from the remote_names since calibration won't get through it (which raises an issue with\n", " # hybrid models). We however still need to include the associated module's forward pass in\n", " # the hybrid model\n", - " if isinstance(module, (Conv1D, Embedding)):\n", + " # Also include the embedding and language model head as they represent a lot of the model's\n", + " # parameters\n", + " if isinstance(module, (Conv1D, Embedding)) or \"lm_head\" in name:\n", " remote_names.append(name)\n", "\n", " elif isinstance(module, CustomConv1D):\n", @@ -255,7 +257,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 113, "metadata": {}, "outputs": [], "source": [ @@ -264,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -280,9 +282,24 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Training Progress: 0%| | 0/100 [3:52:58 6\u001b[0m \u001b[43mtrain_custom_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhybrid_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraining_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfhe\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdisable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[24], line 37\u001b[0m, in \u001b[0;36mtrain_custom_model\u001b[0;34m(hybrid_model, train_dataloader, training_args, fhe)\u001b[0m\n\u001b[1;32m 33\u001b[0m run_optimizer \u001b[38;5;241m=\u001b[39m is_last_batch_step \u001b[38;5;129;01mor\u001b[39;00m accumulate_gradients\n\u001b[1;32m 35\u001b[0m hybrid_model\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mtoggle_run_optimizer(enable\u001b[38;5;241m=\u001b[39mrun_optimizer)\n\u001b[0;32m---> 37\u001b[0m loss, grad_norm \u001b[38;5;241m=\u001b[39m \u001b[43mhybrid_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minput_ids\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlabels\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfhe\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfhe\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 39\u001b[0m total_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m 41\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m grad_norm \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", - "File \u001b[0;32m~/Documents/concrete-ml/src/concrete/ml/torch/hybrid_model.py:413\u001b[0m, in \u001b[0;36mHybridFHEModel.__call__\u001b[0;34m(self, x, fhe)\u001b[0m\n\u001b[1;32m 403\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Call method to run the model locally with a fhe mode.\u001b[39;00m\n\u001b[1;32m 404\u001b[0m \n\u001b[1;32m 405\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 410\u001b[0m \u001b[38;5;124;03m (torch.Tensor): The output tensor.\u001b[39;00m\n\u001b[1;32m 411\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 412\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mset_fhe_mode(fhe)\n\u001b[0;32m--> 413\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 414\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m x\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1539\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1540\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1544\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/use_case_examples/lora_finetune/lora_module.py:32\u001b[0m, in \u001b[0;36mLoraTraining.forward\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 29\u001b[0m x, y \u001b[38;5;241m=\u001b[39m inputs\n\u001b[1;32m 31\u001b[0m \u001b[38;5;66;03m# some parts on server side\u001b[39;00m\n\u001b[0;32m---> 32\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minference_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43my\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 34\u001b[0m loss \u001b[38;5;241m=\u001b[39m outputs\u001b[38;5;241m.\u001b[39mloss\n\u001b[1;32m 35\u001b[0m loss \u001b[38;5;241m=\u001b[39m loss \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgradient_accumulation_steps\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1539\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1540\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1544\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/peft/peft_model.py:1430\u001b[0m, in \u001b[0;36mPeftModelForCausalLM.forward\u001b[0;34m(self, input_ids, attention_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict, task_ids, **kwargs)\u001b[0m\n\u001b[1;32m 1428\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_enable_peft_forward_hooks(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 1429\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m {k: v \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mitems() \u001b[38;5;28;01mif\u001b[39;00m k \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mspecial_peft_forward_args}\n\u001b[0;32m-> 1430\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbase_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1431\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1432\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1433\u001b[0m \u001b[43m \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1434\u001b[0m \u001b[43m \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1435\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1436\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1439\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1441\u001b[0m batch_size \u001b[38;5;241m=\u001b[39m _get_batch_size(input_ids, inputs_embeds)\n\u001b[1;32m 1442\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attention_mask \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 1443\u001b[0m \u001b[38;5;66;03m# concat prompt attention mask\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1539\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1540\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1544\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/peft/tuners/tuners_utils.py:179\u001b[0m, in \u001b[0;36mBaseTuner.forward\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any):\n\u001b[0;32m--> 179\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py:1302\u001b[0m, in \u001b[0;36mGPT2LMHeadModel.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m 1294\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1295\u001b[0m \u001b[38;5;124;03mlabels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):\u001b[39;00m\n\u001b[1;32m 1296\u001b[0m \u001b[38;5;124;03m Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set\u001b[39;00m\n\u001b[1;32m 1297\u001b[0m \u001b[38;5;124;03m `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`\u001b[39;00m\n\u001b[1;32m 1298\u001b[0m \u001b[38;5;124;03m are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`\u001b[39;00m\n\u001b[1;32m 1299\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1300\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[0;32m-> 1302\u001b[0m transformer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtransformer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1303\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1304\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1305\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1306\u001b[0m \u001b[43m \u001b[49m\u001b[43mtoken_type_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken_type_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1307\u001b[0m \u001b[43m \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1308\u001b[0m \u001b[43m \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1309\u001b[0m \u001b[43m \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1310\u001b[0m \u001b[43m \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1311\u001b[0m \u001b[43m \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1312\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1313\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1314\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1315\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1316\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1317\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m transformer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m 1319\u001b[0m \u001b[38;5;66;03m# Set device for model parallelism\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1539\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1540\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1544\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py:1116\u001b[0m, in \u001b[0;36mGPT2Model.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m 1104\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m 1105\u001b[0m block\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m 1106\u001b[0m hidden_states,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1113\u001b[0m output_attentions,\n\u001b[1;32m 1114\u001b[0m )\n\u001b[1;32m 1115\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1116\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[43mblock\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1117\u001b[0m \u001b[43m \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1118\u001b[0m \u001b[43m \u001b[49m\u001b[43mlayer_past\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlayer_past\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1119\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1120\u001b[0m \u001b[43m \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1121\u001b[0m \u001b[43m \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1122\u001b[0m \u001b[43m \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1123\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1124\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1125\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1127\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m 1128\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1539\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1540\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1544\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py:651\u001b[0m, in \u001b[0;36mGPT2Block.forward\u001b[0;34m(self, hidden_states, layer_past, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, use_cache, output_attentions)\u001b[0m\n\u001b[1;32m 649\u001b[0m residual \u001b[38;5;241m=\u001b[39m hidden_states\n\u001b[1;32m 650\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mln_2(hidden_states)\n\u001b[0;32m--> 651\u001b[0m feed_forward_hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmlp\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 652\u001b[0m \u001b[38;5;66;03m# residual connection\u001b[39;00m\n\u001b[1;32m 653\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m residual \u001b[38;5;241m+\u001b[39m feed_forward_hidden_states\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1539\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1540\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1544\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py:571\u001b[0m, in \u001b[0;36mGPT2MLP.forward\u001b[0;34m(self, hidden_states)\u001b[0m\n\u001b[1;32m 570\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, hidden_states: Optional[Tuple[torch\u001b[38;5;241m.\u001b[39mFloatTensor]]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m torch\u001b[38;5;241m.\u001b[39mFloatTensor:\n\u001b[0;32m--> 571\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mc_fc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 572\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mact(hidden_states)\n\u001b[1;32m 573\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mc_proj(hidden_states)\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1539\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1540\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1544\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/use_case_examples/lora_finetune/remote_module.py:49\u001b[0m, in \u001b[0;36mCustomConv1D.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m---> 49\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mForwardBackwardModule\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward_module\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward_module\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/.venv/lib/python3.10/site-packages/torch/autograd/function.py:598\u001b[0m, in \u001b[0;36mFunction.apply\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m 595\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch\u001b[38;5;241m.\u001b[39m_C\u001b[38;5;241m.\u001b[39m_are_functorch_transforms_active():\n\u001b[1;32m 596\u001b[0m \u001b[38;5;66;03m# See NOTE: [functorch vjp and autograd interaction]\u001b[39;00m\n\u001b[1;32m 597\u001b[0m args \u001b[38;5;241m=\u001b[39m _functorch\u001b[38;5;241m.\u001b[39mutils\u001b[38;5;241m.\u001b[39munwrap_dead_wrappers(args)\n\u001b[0;32m--> 598\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 600\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_setup_ctx_defined:\n\u001b[1;32m 601\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 602\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIn order to use an autograd.Function with functorch transforms \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 603\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m(vmap, grad, jvp, jacrev, ...), it must override the setup_context \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 604\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstaticmethod. For more details, please see \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 605\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://pytorch.org/docs/master/notes/extending.func.html\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 606\u001b[0m )\n", - "File \u001b[0;32m~/Documents/concrete-ml/use_case_examples/lora_finetune/remote_module.py:30\u001b[0m, in \u001b[0;36mForwardBackwardModule.forward\u001b[0;34m(ctx, input, forward_module, backward_module)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(ctx, \u001b[38;5;28minput\u001b[39m, forward_module, backward_module):\n\u001b[1;32m 29\u001b[0m ctx\u001b[38;5;241m.\u001b[39mbackward_module \u001b[38;5;241m=\u001b[39m backward_module\n\u001b[0;32m---> 30\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mforward_module\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m output\n", - "File \u001b[0;32m~/Documents/concrete-ml/src/concrete/ml/torch/hybrid_model.py:254\u001b[0m, in \u001b[0;36mRemoteModule.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfhe_local_mode \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m {\n\u001b[1;32m 246\u001b[0m HybridFHEMode\u001b[38;5;241m.\u001b[39mCALIBRATE,\n\u001b[1;32m 247\u001b[0m HybridFHEMode\u001b[38;5;241m.\u001b[39mREMOTE,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 250\u001b[0m }:\n\u001b[1;32m 251\u001b[0m \u001b[38;5;66;03m# Using quantized module\u001b[39;00m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprivate_q_module \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 253\u001b[0m y \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mTensor(\n\u001b[0;32m--> 254\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprivate_q_module\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdetach\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnumpy\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfhe\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfhe_local_mode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 255\u001b[0m )\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfhe_local_mode \u001b[38;5;241m==\u001b[39m HybridFHEMode\u001b[38;5;241m.\u001b[39mCALIBRATE:\n\u001b[1;32m 258\u001b[0m \u001b[38;5;66;03m# Calling torch + gathering calibration data\u001b[39;00m\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprivate_module \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/src/concrete/ml/quantization/quantized_module.py:479\u001b[0m, in \u001b[0;36mQuantizedModule.forward\u001b[0;34m(self, fhe, debug, *x)\u001b[0m\n\u001b[1;32m 476\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdequantize_output(\u001b[38;5;241m*\u001b[39mto_tuple(q_y_pred))\n\u001b[1;32m 477\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m y_pred, debug_value_tracker\n\u001b[0;32m--> 479\u001b[0m q_y_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquantized_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mq_x\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfhe\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfhe\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 481\u001b[0m \u001b[38;5;66;03m# De-quantize the output predicted values\u001b[39;00m\n\u001b[1;32m 482\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdequantize_output(\u001b[38;5;241m*\u001b[39mto_tuple(q_y_pred))\n", - "File \u001b[0;32m~/Documents/concrete-ml/src/concrete/ml/quantization/quantized_module.py:520\u001b[0m, in \u001b[0;36mQuantizedModule.quantized_forward\u001b[0;34m(self, fhe, *q_x)\u001b[0m\n\u001b[1;32m 512\u001b[0m assert_true(\n\u001b[1;32m 513\u001b[0m n_values \u001b[38;5;241m==\u001b[39m n_inputs,\n\u001b[1;32m 514\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGot \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mn_values\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m inputs, expected \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mn_inputs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. Either the quantized module has not been \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 515\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproperly initialized or the input data has been changed since its initialization.\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 516\u001b[0m \u001b[38;5;167;01mValueError\u001b[39;00m,\n\u001b[1;32m 517\u001b[0m )\n\u001b[1;32m 519\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m fhe \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdisable\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_clear_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mq_x\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 521\u001b[0m simulate \u001b[38;5;241m=\u001b[39m fhe \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msimulate\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 522\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fhe_forward(\u001b[38;5;241m*\u001b[39mq_x, simulate\u001b[38;5;241m=\u001b[39msimulate)\n", - "File \u001b[0;32m~/Documents/concrete-ml/src/concrete/ml/quantization/quantized_module.py:563\u001b[0m, in \u001b[0;36mQuantizedModule._clear_forward\u001b[0;34m(self, *q_x)\u001b[0m\n\u001b[1;32m 561\u001b[0m error_tracker: List[\u001b[38;5;28mint\u001b[39m] \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 562\u001b[0m layer\u001b[38;5;241m.\u001b[39merror_tracker \u001b[38;5;241m=\u001b[39m error_tracker\n\u001b[0;32m--> 563\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mlayer\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 564\u001b[0m layer\u001b[38;5;241m.\u001b[39merror_tracker \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 566\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(error_tracker) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 567\u001b[0m \u001b[38;5;66;03m# The error message contains the ONNX tensor name that\u001b[39;00m\n\u001b[1;32m 568\u001b[0m \u001b[38;5;66;03m# triggered this error\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/concrete-ml/src/concrete/ml/quantization/base_quantized_op.py:391\u001b[0m, in \u001b[0;36mQuantizedOp.__call__\u001b[0;34m(self, *q_inputs)\u001b[0m\n\u001b[1;32m 379\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39mq_inputs: ONNXOpInputOutputType) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ONNXOpInputOutputType:\n\u001b[1;32m 380\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Process the forward pass of the quantized op according to the implementation.\u001b[39;00m\n\u001b[1;32m 381\u001b[0m \n\u001b[1;32m 382\u001b[0m \u001b[38;5;124;03m The calibrate method needs to be called with sample data before using this function.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 388\u001b[0m \u001b[38;5;124;03m ONNXOpInputOutputType: Quantized output.\u001b[39;00m\n\u001b[1;32m 389\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 391\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mq_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mq_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/concrete-ml/src/concrete/ml/quantization/quantized_ops.py:375\u001b[0m, in \u001b[0;36mQuantizedGemm.q_impl\u001b[0;34m(self, calibrate_rounding, *q_inputs, **attrs)\u001b[0m\n\u001b[1;32m 370\u001b[0m input2_q_values_copy \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 371\u001b[0m copy_function(input2_q_values) \u001b[38;5;28;01mif\u001b[39;00m is_encrypted_gemm \u001b[38;5;28;01melse\u001b[39;00m input2_q_values\n\u001b[1;32m 372\u001b[0m )\n\u001b[1;32m 374\u001b[0m \u001b[38;5;66;03m# Core matmul operation in full integers with a shape change (INTEGERS)\u001b[39;00m\n\u001b[0;32m--> 375\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tag(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mop_instance_name \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.matmul\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m 376\u001b[0m \u001b[38;5;66;03m# We implement our own encrypted matmul to be able to round before PBS\u001b[39;00m\n\u001b[1;32m 377\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_encrypted_gemm:\n\u001b[1;32m 378\u001b[0m matmul \u001b[38;5;241m=\u001b[39m matmul(input1_q_values_copy, input2_q_values_copy)\n", - "File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/contextlib.py:139\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[0;34m(self, typ, value, traceback)\u001b[0m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgenerator didn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt yield\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 139\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__exit__\u001b[39m(\u001b[38;5;28mself\u001b[39m, typ, value, traceback):\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m typ \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "name": "stderr", + "output_type": "stream", + "text": [ + "Training Progress: 100%|██████████| 2/2 [10:17<00:00, 308.74s/it]\n" ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2AAAAIjCAYAAABlKXjSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACJWUlEQVR4nOzdd3gU5cL+8e9uKgmEEEILBBJ6MwGlSAkJEMCAQbChIl2KJbSAR1AREA+CNJViQ5AmTUBUpJNQpEPohN5Dl1QSUvb3hz/ynkjLQmBS7s91cV3vzj47c0/yvBxu55lZk8VisSAiIiIiIiKPndnoACIiIiIiInmFCpiIiIiIiMgTogImIiIiIiLyhKiAiYiIiIiIPCEqYCIiIiIiIk+ICpiIiIiIiMgTogImIiIiIiLyhKiAiYiIiIiIPCEqYCIiIiIiIk+ICpiIiDxRp06dwmQyMX369Czd79ChQzGZTFm6z+wuICCAgIAAo2M8Fo86T0wmE0OHDs3STCIiWUEFTEQki02fPh2TycSOHTuMjnJftwvLvf5cvHjR6Ih3SEhIYOjQoYSFhRkdJYP//bnZ2tri5ubGM888Q58+fTh48KDR8bLUg+bN7T+5tRiKiDwqW6MDiIiIsaZMmUL+/Pnv2O7q6vrkwzxAQkICw4YNA7jjH/gfffQRH3zwgQGp/tGsWTM6duyIxWIhOjqaPXv28NNPPzF58mRGjRpF//79s/yYK1euzPJ9PsiLL75I+fLl01/HxcXx9ttv07ZtW1588cX07cWKFXuk45QpU4abN29iZ2f3UJ+/efMmtrb6Z46IZD/6m0lEJI97+eWXcXd3NzrGI7O1tTX0H9wVK1bkzTffzLDt888/Jzg4mNDQUCpXrkzLli2z5FgJCQk4OTlhb2+fJfuzho+PDz4+Pumvr169yttvv42Pj88d5/+/EhMTsbe3x2zO3OIbk8mEo6PjQ+d8lM+KiDxOWoIoImKQ3bt3ExQUhIuLC/nz56dp06Zs2bIlw5jk5GSGDRtGhQoVcHR0pHDhwjRs2JBVq1alj7l48SJdunShVKlSODg4UKJECV544QVOnTr1yBkvXbqEra1t+lWn/xUZGYnJZGLixInp206cOMErr7yCm5sbTk5OPPvss/zxxx8PPM697mXq3LkzXl5ewD/3BBUpUgSAYcOGpS91u32fz93uAUtJSeHTTz+lXLlyODg44OXlxeDBg0lKSsowzsvLi+eff56NGzdSp04dHB0dKVu2LDNmzHhg9vspXLgwc+fOxdbWls8++yx9++1lqv/+HYWFhWEymTIssQwICKB69ers3LmTRo0a4eTkxODBg9Pf+9+f2+3Pz58/n88++4xSpUrh6OhI06ZNOXbs2B35Jk2aRNmyZcmXLx916tRhw4YNWXJf2e0cc+fO5aOPPqJkyZI4OTkRExPD9evXGTBgAE899RT58+fHxcWFoKAg9uzZk2Efd7sHrHPnzuTPn5/z58/Tpk0b8ufPT5EiRRgwYACpqakZPv/ve8Buz49jx47RuXNnXF1dKViwIF26dCEhISHDZ2/evEnv3r1xd3enQIECtG7dmvPnz+u+MhHJEroCJiJigAMHDuDn54eLiwvvv/8+dnZ2fPvttwQEBBAeHk7dunWBf/7ROHLkSN566y3q1KlDTEwMO3bsYNeuXTRr1gyAl156iQMHDhASEoKXlxeXL19m1apVnDlzJr283M/169fv2GZra4urqyvFihXD39+f+fPn88knn2QYM2/ePGxsbHjllVeAf8pa/fr1SUhIoHfv3hQuXJiffvqJ1q1bs3DhQtq2bftIP7MiRYowZcqUO5a7/e/VmH976623+Omnn3j55ZcJDQ1l69atjBw5kkOHDrF48eIMY48dO8bLL79Mt27d6NSpEz/++COdO3fmmWeeoVq1ag+du3Tp0vj7+7Nu3TpiYmJwcXGxeh/Xrl0jKCiI1157jTfffPOBy/s+//xzzGYzAwYMIDo6mtGjR9O+fXu2bt2aPmbKlCm89957+Pn50a9fP06dOkWbNm0oVKgQpUqVsjrj3Xz66afY29szYMAAkpKSsLe35+DBgyxZsoRXXnkFb29vLl26xLfffou/vz8HDx7Ew8PjvvtMTU2lRYsW1K1blzFjxrB69WrGjh1LuXLlePvttx+Y6dVXX8Xb25uRI0eya9cufvjhB4oWLcqoUaPSx3Tu3Jn58+fToUMHnn32WcLDw2nVqtUj/zxERACwiIhIlpo2bZoFsGzfvv2eY9q0aWOxt7e3HD9+PH3bhQsXLAUKFLA0atQofZuvr6+lVatW99zP33//bQEsX3zxhdU5P/nkEwtw1z+VKlVKH/ftt99aAMu+ffsyfL5q1aqWJk2apL/u27evBbBs2LAhfVtsbKzF29vb4uXlZUlNTbVYLBbLyZMnLYBl2rRp6eP8/f0t/v7+d2Ts1KmTpUyZMumvr1y5YgEsn3zyyT3P57aIiAgLYHnrrbcyjBswYIAFsKxduzZ9W5kyZSyAZf369enbLl++bHFwcLCEhobecax/AyzvvvvuPd/v06ePBbDs2bPHYrH83xw5efJkhnHr1q2zAJZ169alb/P397cAlm+++eaO/f7753b781WqVLEkJSWlb//yyy8z/A6TkpIshQsXttSuXduSnJycPm769OkW4K6/i3u52+/kdo6yZctaEhISMoxPTExMnwu3nTx50uLg4GAZPnx4hm3/niedOnWyABnGWSwWS82aNS3PPPNMhm3/znR7fnTt2jXDuLZt21oKFy6c/nrnzp0WwNK3b98M4zp37nzPuSciYg0tQRQRecJSU1NZuXIlbdq0oWzZsunbS5QowRtvvMHGjRuJiYkB/nkQxoEDBzh69Ohd95UvXz7s7e0JCwvj77//fqg8v/zyC6tWrcrwZ9q0aenvv/jii9ja2jJv3rz0bfv37+fgwYO0a9cufduyZcuoU6cODRs2TN+WP39+evTowalTp5740wCXLVsGcMfDL0JDQwHuWBpZtWpV/Pz80l8XKVKESpUqceLEiUfOcvshJ7GxsQ/1eQcHB7p06ZLp8V26dMlwf9jt87p9Ljt27ODatWt07949w31z7du3p1ChQg+V8W46depEvnz5MmxzcHBIvw8sNTWVa9eukT9/fipVqsSuXbsytd9evXpleO3n55fp39PdPnvt2rX0/59bvnw5AO+8806GcSEhIZnav4jIg6iAiYg8YVeuXCEhIYFKlSrd8V6VKlVIS0vj7NmzAAwfPpwbN25QsWJFnnrqKQYOHMjevXvTxzs4ODBq1Cj+/PNPihUrRqNGjRg9erRVj5Bv1KgRgYGBGf7Uq1cv/X13d3eaNm3K/Pnz07fNmzcPW1vbDE+9O3369D3P6fb7T9Lp06cxm80ZntgHULx4cVxdXe/IU7p06Tv2UahQoYcutv8rLi4OgAIFCjzU50uWLGnVAzf+fS63S9Xtc7l97v/+2dja2mZq2WpmeXt737EtLS2N8ePHU6FCBRwcHHB3d6dIkSLs3buX6OjoB+7T0dEx/V7A26z5PWXmZ2M2m+/I/u+flYjIw1IBExHJxho1asTx48f58ccfqV69Oj/88ANPP/00P/zwQ/qYvn37cuTIEUaOHImjoyMff/wxVapUYffu3VmW47XXXuPIkSNEREQAMH/+fJo2bZplT0+81xco//vBClm573+zsbG563aLxfLIGfbv34+NjU36P+qtPd9/X0V6kMd5Lta4W+7//ve/9O/fn0aNGjFr1ixWrFjBqlWrqFatGmlpaQ/c573OLbOyy89GRPIuFTARkSesSJEiODk5ERkZecd7hw8fxmw24+npmb7Nzc2NLl268PPPP3P27Fl8fHzueBJbuXLlCA0NZeXKlezfv59bt24xduzYLMvcpk0b7O3tmTdvHhERERw5coTXXnstw5gyZcrc85xuv38vhQoV4saNG3ds//dVqsyWqdvHS0tLu2P55qVLl7hx48Z982SlM2fOEB4eTr169dKvgN2+6vLvc35SVwlvn/u/n4yYkpKSJU/PvJ+FCxfSuHFjpk6dymuvvUbz5s0JDAy86+/fCLfnzcmTJzNsv9tTJEVEHoYKmIjIE2ZjY0Pz5s359ddfM/xj99KlS8yZM4eGDRumPynv2rVrGT6bP39+ypcvn/4Y9YSEBBITEzOMKVeuHAUKFLjjUeuPwtXVlRYtWjB//nzmzp2Lvb09bdq0yTCmZcuWbNu2jc2bN6dvi4+P57vvvsPLy4uqVavec//lypXj8OHDXLlyJX3bnj172LRpU4ZxTk5OwJ3F5W5uf+fWhAkTMmwfN24cwBN5qt3169d5/fXXSU1N5cMPP0zfXq5cOQDWr1+fvi01NZXvvvvusWcCqFWrFoULF+b7778nJSUlffvs2bOzZMnl/djY2NxxtWnBggWcP3/+sR43s1q0aAHA5MmTM2z/+uuvjYgjIrmQHkMvIvKY/Pjjj+k39P+vPn36MGLECFatWkXDhg155513sLW15dtvvyUpKYnRo0enj61atSoBAQE888wzuLm5sWPHDhYuXMh7770HwJEjR2jatCmvvvoqVatWxdbWlsWLF3Pp0qU7rlDdy8KFC9MfEvG/mjVrluFx5+3atePNN99k8uTJtGjRAldX1wzjP/jgA37++WeCgoLo3bs3bm5u/PTTT5w8eZJffvnlvl/A27VrV8aNG0eLFi3o1q0bly9f5ptvvqFatWrpD0eAf5a0Va1alXnz5lGxYkXc3NyoXr061atXv2Ofvr6+dOrUie+++44bN27g7+/Ptm3b+Omnn2jTpg2NGzfO1M8ns44cOcKsWbOwWCzExMSwZ88eFixYQFxcHOPGjeO5555LH1utWjWeffZZBg0axPXr13Fzc2Pu3LkZytDjZG9vz9ChQwkJCaFJkya8+uqrnDp1iunTp1OuXDmrrjRa6/nnn2f48OF06dKF+vXrs2/fPmbPnp3hgTRGeuaZZ3jppZeYMGEC165dS38M/ZEjRwDrrsKKiNyNCpiIyGMyZcqUu27v3Lkz1apVY8OGDQwaNIiRI0eSlpZG3bp1mTVrVvp3gAH07t2bpUuXsnLlSpKSkihTpgwjRoxg4MCBAHh6evL666+zZs0aZs6cia2tLZUrV2b+/Pm89NJLmcp5r+9OWrduXYYC1rp1a/Lly0dsbGyGpx/eVqxYMf766y/+85//8PXXX5OYmIiPjw+//fbbA682ValShRkzZjBkyBD69+9P1apVmTlzJnPmzMnwpcQAP/zwAyEhIfTr149bt27xySef3LWA3R5btmxZpk+fzuLFiylevDiDBg264zvNssLtJ0iazWZcXFzw9vamU6dO9OjR465X/2bPnk3Pnj35/PPPcXV1pVu3bjRu3Dj9+90et/feew+LxcLYsWMZMGAAvr6+LF26lN69e+Po6PjYjjt48GDi4+OZM2cO8+bN4+mnn+aPP/7ggw8+eGzHtNaMGTMoXrw4P//8M4sXLyYwMJB58+ZRqVKlx/qzEZG8wWTRXaciIiLCP08oLFKkCC+++CLff/+90XGylYiICGrWrMmsWbNo37690XFEJAfTPWAiIiJ5UGJi4h33Ys2YMYPr168TEBBgTKhs4ubNm3dsmzBhAmazmUaNGhmQSERyEy1BFBERyYO2bNlCv379eOWVVyhcuDC7du1i6tSpVK9enVdeecXoeIYaPXo0O3fupHHjxtja2vLnn3/y559/0qNHjwxPKBUReRhagigiIpIHnTp1it69e7Nt27b0B4G0bNmSzz//nKJFixodz1CrVq1i2LBhHDx4kLi4OEqXLk2HDh348MMPsbXVf7sWkUejAiYiIiIiIvKE6B4wERERERGRJ0QFTERERERE5AnRQuaHlJaWxoULFyhQoIC+lFFEREREJA+zWCzExsbi4eGB2Xz/a1wqYA/pwoULehKSiIiIiIikO3v2LKVKlbrvGEML2Pr16/niiy/YuXMnUVFRLF68mDZt2tz3M7Nnz2b06NEcPXqUggULEhQUxBdffEHhwoXvGDt37lxef/11XnjhBZYsWZK+PS4ujg8++IAlS5Zw7do1vL296d27N7169cp09gIFCgD//JBdXFwy/bnHITk5mZUrV9K8eXPs7OwMzSI5g+aMWEtzRqylOSPW0pwRa2WnORMTE4Onp2d6R7gfQwtYfHw8vr6+dO3alRdffPGB4zdt2kTHjh0ZP348wcHBnD9/nl69etG9e3cWLVqUYeypU6cYMGAAfn5+d+ynf//+rF27llmzZuHl5cXKlSt555138PDwoHXr1pnKfnvZoYuLS7YoYE5OTri4uBg++SRn0JwRa2nOiLU0Z8RamjNirew4ZzJza5KhD+EICgpixIgRtG3bNlPjN2/ejJeXF71798bb25uGDRvSs2dPtm3blmFcamoq7du3Z9iwYZQtW/aO/fz111906tSJgIAAvLy86NGjB76+vnfsR0REREREJCvlqHvA6tWrx+DBg1m2bBlBQUFcvnyZhQsX0rJlywzjhg8fTtGiRenWrRsbNmy4Yz/169dn6dKldO3aFQ8PD8LCwjhy5Ajjx4+/57GTkpJISkpKfx0TEwP807yTk5Oz6Awfzu3jG51Dcg7NGbGW5oxYS3NGrKU5I9bKTnPGmgw5qoA1aNCA2bNn065dOxITE0lJSSE4OJhJkyalj9m4cSNTp04lIiLinvv5+uuv6dGjB6VKlcLW1haz2cz3339Po0aN7vmZkSNHMmzYsDu2r1y5Eicnp0c6r6yyatUqoyNIDqM5I9bSnBFrac6ItTRnxFrZYc4kJCRkemyOKmAHDx6kT58+DBkyhBYtWhAVFcXAgQPp1asXU6dOJTY2lg4dOvD999/j7u5+z/18/fXXbNmyhaVLl1KmTBnWr1/Pu+++i4eHB4GBgXf9zKBBg+jfv3/669s32jVv3jxb3AO2atUqmjVrlm3Wv0r2pjkj1tKcEWtpzoi1NGfEWtlpztxeHZcZOaqAjRw5kgYNGjBw4EAAfHx8cHZ2xs/PjxEjRnDp0iVOnTpFcHBw+mfS0tIAsLW1JTIyEg8PDwYPHszixYtp1apV+n4iIiIYM2bMPQuYg4MDDg4Od2y3s7Mz/Bd+W3bKIjmD5oxYS3NGrKU5I9bSnBFrZYc5Y83xc1QBS0hIwNY2Y2QbGxvgny8/q1y5Mvv27cvw/kcffURsbCxffvklnp6eJCYmkpycfMcXpNnY2KSXNRERERERkcfB0AIWFxfHsWPH0l+fPHmSiIgI3NzcKF26NIMGDeL8+fPMmDEDgODgYLp3786UKVPSlyD27duXOnXq4OHhAUD16tUzHMPV1TXDdnt7e/z9/Rk4cCD58uWjTJkyhIeHM2PGDMaNG/cEzlpERERERPIqQwvYjh07aNy4cfrr2/dYderUienTpxMVFcWZM2fS3+/cuTOxsbFMnDiR0NBQXF1dadKkCaNGjbLquHPnzmXQoEG0b9+e69evU6ZMGT777DOrvohZRERERETEWoYWsICAACwWyz3fnz59+h3bQkJCCAkJyfQx7raP4sWLM23atEzvQ0REREREJCsY+kXMIiIiIiIieYkKmIiIiIiIyBOiAiYiIiIiIvKEqICJiIiIiIg8ISpgOVxqmoWtJ6+z86qJrSevk5p274eaiIiIiIiIsXLUFzFLRsv3RzHst4NERScCNsw4uoMSBR35JLgqz1UvYXQ8ERERERH5F10By6GW74/i7Vm7/n/5+j8XoxN5e9Yulu+PMiiZiIiIiIjciwpYDpSaZmHYbwe522LD29uG/XZQyxFFRERERLIZFbAcaNvJ63dc+fpfFiAqOpFtJ68/uVAiIiIiIvJAKmA50OXYe5evhxknIiIiIiJPhgpYDlS0gGOmxh2OisVi0TJEEREREZHsQgUsB6rj7UaJgo6YHjBuSvhxXvlmMwcuRD+RXCIiIiIicn8qYDmQjdnEJ8FVAe4oYab//6dNjZI42duw4/TfBH+9kSG/7ic6IflJRxURERERkf+hApZDPVe9BFPefJriBTMuRyxe0JEpbz7NhNdqsCbUn+d9SpBmgRmbT9N4bBjztp8hTU9HFBERERExhL6IOQd7rnoJmlUtzuZjl1m5YSvN/epSr3xRbMz/XBcrUTAfE994mjfqXOWTpQc4ejmO//yyjznbzvLpC9XwKeVq7AmIiIiIiOQxugKWw9mYTdT1duMZdwt1vd3Sy9f/ql/enWV9/PioVRXyO9iy5+wNXpi0iUGL9nE9/pYBqUVERERE8iYVsDzCzsbMW35lWRvqT9uaJbFY4OdtZ2gyNoxZW07rS5tFRERERJ4AFbA8pqiLI+Pb1WB+z3pULl6AGwnJfLRkPy9M2sjO038bHU9EREREJFdTAcuj6ni78XtIQ4YGV6WAoy37z8fw0pS/GLhgD1fjkoyOJyIiIiKSK6mA5WG2NmY6N/Bm3YAAXnmmFAALdp6j8Zgwpm86SUpqmsEJRURERERyFxUwwT2/A1+84suid+pTvaQLsYkpDP3tIM9/vZFtJ68bHU9EREREJNdQAZN0T5cuxK/vNmREm+oUzGfH4YuxvPrtZvrNi+ByTKLR8UREREREcjwVMMnAxmzizWfLsG5AAK/XKY3JBIt3n6fJ2HB+2HCCZC1LFBERERF5aCpgclduzvaMfPEplrzTAF9PV+KSUhjxxyFafrmBv45fNTqeiIiIiEiOpAIm9+Xr6crit+sz6qWncHO25+jlON74fivvzdlFVPRNo+OJiIiIiOQoKmDyQGaziXa1S7M21J+O9cpgNsHve6NoOjacKWHHuZWiZYkiIiIiIpmhAiaZ5upkz/AXqvNbSEOeKVOIhFupjFp+mOcmrGf9kStGxxMRERERyfZUwMRq1TwKsrBXPca+4ot7fgdOXI2n44/b6DVzJ+f+TjA6noiIiIhItqUCJg/FZDLx0jOlWDvAn64NvLExm1h+4CKB48L5es1REpNTjY4oIiIiIpLtqIDJI3FxtGNIcFX+6N2QOt5uJCanMXbVEVpMWM/aw5eMjiciIiIikq2ogEmWqFzchXk9nuXL12pQzMWB09cS6Dp9B2/9tJ0z17QsUUREREQEVMAkC5lMJl6oUZI1oQH0bFQWW7OJ1YcuEzg+nHGrjmhZooiIiIjkeSpgkuXyO9gyqGUVlvf1o0H5wtxKSeOrNUcJHBfOigMXsVgsRkcUERERETGECpg8NuWLFmBWt7pMbv80HgUdOff3TXrO3Ennads5eTXe6HgiIiIiIk+cCpg8ViaTiZZPlWB1qD/vNi6HvY2Z8CNXaDF+PaOXHybhVorREUVEREREnhgVMHkinOxtGdiiMiv6NcK/YhFupaYxOew4gWPDWbYvSssSRURERCRPUAGTJ8rb3ZnpXWrzXYdnKFUoHxeiE3ln9i46TN3GscuxRscTEREREXmsVMDkiTOZTDSvVpzV/f3p07QC9rZmNh67ynMTNvDfZYeIS9KyRBERERHJnVTAxDCOdjb0a1aR1f38CaxSlJQ0C9+tP0HTsWH8GnFeyxJFREREJNdRARPDlS7sxA+davNj51qUKezEpZgk+syN4LXvthB5UcsSRURERCT3UAGTbKNJ5WKs6NuIAc0r4mhnZuvJ67T8agPDfjtATGKy0fFERERERB6ZCphkK452NrzXpAKr+/sTVL04qWkWpm06RZMxYSzceY60NC1LFBEREZGcSwVMsqVShZyY8uYzzOhah7JFnLkad4sBC/bwyrebOXAh2uh4IiIiIiIPRQVMsrVGFYuwvE8jPgiqjJO9DTtP/03w1xsZ8ut+ohO0LFFEREREchYVMMn27G3N9PIvx5pQf573KUGaBWZsPk3jsWHM235GyxJFREREJMdQAZMco0TBfEx842nmdK9LhaL5uR5/i//8so+2U/5i77kbRscTEREREXkgFTDJceqXc2dZHz8+alWF/A627Dl7gxcmbWLQor1cj79ldDwRERERkXtSAZMcyc7GzFt+ZVkb6k/bmiWxWODnbWdpMjaMWVtOk6pliSIiIiKSDamASY5W1MWR8e1qML9nPSoXL8CNhGQ+WrKfFyZtZOfpv42OJyIiIiKSgaEFbP369QQHB+Ph4YHJZGLJkiUP/Mzs2bPx9fXFycmJEiVK0LVrV65du3bXsXPnzsVkMtGmTZs73jt06BCtW7emYMGCODs7U7t2bc6cOfOIZyRGqePtxu8hDRnWuhoFHG3Zfz6Gl6b8xcAFe7gal2R0PBERERERwOACFh8fj6+vL5MmTcrU+E2bNtGxY0e6devGgQMHWLBgAdu2baN79+53jD116hQDBgzAz8/vjveOHz9Ow4YNqVy5MmFhYezdu5ePP/4YR0fHRz4nMY6tjZlO9b1YNyCAV2uVAmDBznM0HhPG9E0nSUlNMzihiIiIiOR1tkYePCgoiKCgoEyP37x5M15eXvTu3RsAb29vevbsyahRozKMS01NpX379gwbNowNGzZw48aNDO9/+OGHtGzZktGjR6dvK1eu3MOfiGQr7vkdGP2yL6/VKc2QX/ez/3wMQ387yNztZxn+QnXqeLsZHVFERERE8ihDC5i16tWrx+DBg1m2bBlBQUFcvnyZhQsX0rJlywzjhg8fTtGiRenWrRsbNmzI8F5aWhp//PEH77//Pi1atGD37t14e3szaNCguy5VvC0pKYmkpP9byhYTEwNAcnIyycnGfiHw7eMbnSO7eapEfhb2qMu8HecYt/oohy/G8uq3m3nBtwTvt6hI0QIORkc0jOaMWEtzRqylOSPW0pwRa2WnOWNNBpPFYskWj4szmUwsXrz4viUIYMGCBXTt2pXExERSUlIIDg7ml19+wc7ODoCNGzfy2muvERERgbu7O507d+bGjRvp95ddvHiREiVK4OTkxIgRI2jcuDHLly9n8ODBrFu3Dn9//7sed+jQoQwbNuyO7XPmzMHJyemRzl0ev7hk+OOMmc2XTVgw4WBjIahUGo2KW7DRo2hERERE5BEkJCTwxhtvEB0djYuLy33H5qgCdvDgQQIDA+nXrx8tWrQgKiqKgQMHUrt2baZOnUpsbCw+Pj5Mnjw5fWnjvwvYhQsXKFmyJK+//jpz5sxJ33fr1q1xdnbm559/vuux73YFzNPTk6tXrz7wh/y4JScns2rVKpo1a5ZeROXu9p2PZujvh9h77p8rmOWLOPPJ81V4tmzeWpaoOSPW0pwRa2nOiLU0Z8Ra2WnOxMTE4O7unqkClqOWII4cOZIGDRowcOBAAHx8fHB2dsbPz48RI0Zw6dIlTp06RXBwcPpn0tL+efCCra0tkZGReHp6YmtrS9WqVTPsu0qVKmzcuPGex3ZwcMDB4c4la3Z2dob/wm/LTlmyq6e93FnyTkMW7DzLqOWRHLsST4dpO2jlU4KPWlWhRMF8Rkd8ojRnxFqaM2ItzRmxluaMWCs7zBlrjp+jClhCQgK2thkj29jYAGCxWKhcuTL79u3L8P5HH31EbGwsX375JZ6entjb21O7dm0iIyMzjDty5AhlypR5vCcg2YLZbKJd7dI8V60EY1dFMmvLaf7YG8W6w5cJaVKBbg29sbfVukQRERERyXqGFrC4uDiOHTuW/vrkyZNERETg5uZG6dKlGTRoEOfPn2fGjBkABAcH0717d6ZMmZK+BLFv377UqVMHDw8PAKpXr57hGK6urndsHzhwIO3ataNRo0bp94D99ttvhIWFPd4TlmyloJMdw1+oTrvannzy6wF2nP6bUcsPs2DHWYa2rkajikWMjigiIiIiuYyh/5l/x44d1KxZk5o1awLQv39/atasyZAhQwCIiorK8OXInTt3Zty4cUycOJHq1avzyiuvUKlSJRYtWmTVcdu2bcs333zD6NGjeeqpp/jhhx/45ZdfaNiwYdadnOQY1TwKsqBXPca+4ot7fgdOXI2n44/b6DVzJ+f+TjA6noiIiIjkIoZeAQsICOB+zwCZPn36HdtCQkIICQnJ9DHutg+Arl270rVr10zvR3I3k8nES8+Uolm1YkxYdZSfNp9i+YGLhB25zLsB5eneqCyOdjZGxxQRERGRHE43uoj8DxdHO4YEV+WP3g2p6+1GYnIaY1cdocWE9aw9fMnoeCIiIiKSw6mAidxF5eIuzO3xLF++VoNiLg6cvpZA1+k7eOun7Zy5pmWJIiIiIvJwVMBE7sFkMvFCjZKsCQ2gZ6Oy2JpNrD50mcDx4YxbdYTE5FSjI4qIiIhIDqMCJvIA+R1sGdSyCsv7NqJheXdupaTx1ZqjBI4LZ8WBi/e9j1FERERE5H+pgIlkUvmi+ZnZrQ6T2z+NR0FHzv19k54zd9J52nZOXIkzOp6IiIiI5AAqYCJWMJlMtHyqBKtD/Xm3cTnsbcyEH7nCcxM2MHr5YRJupRgdUURERESyMRUwkYfgZG/LwBaVWdGvEQGVinArNY3JYcdpOjacP/ZGaVmiiIiIiNyVCpjII/B2d2Za59p81+EZShXKR1R0Iu/O2cWbU7dy7HKs0fFEREREJJtRARN5RCaTiebVirO6vz99mlbA3tbMpmPXeG7CBv677BBxSVqWKCIiIiL/UAETySKOdjb0a1aR1f38CaxSjJQ0C9+tP0GTMWH8GnFeyxJFRERERAVMJKuVLuzED51qMa1zbcoUduJybBJ95kbQ7rstHL4YY3Q8ERERETGQCpjIY9K4clFW9G3EgOYVcbQzs+3kdVp9tZFhvx0gJjHZ6HgiIiIiYgAVMJHHyNHOhveaVGB1f3+CqhcnNc3CtE2naDImjIU7z5GWpmWJIiIiInmJCpjIE1CqkBNT3nyGGV3rULaIM1fjbjFgwR5e+XYz+89HGx1PRERERJ4QFTCRJ6hRxSIs79OID4Iq42Rvw87Tf9N64kY+XrKf6AQtSxQRERHJ7VTARJ4we1szvfzLsSbUn+d9SpBmgZlbTtN4bBhzt53RskQRERGRXEwFTMQgJQrmY+IbTzOne10qFM3P9fhbfLBoH22n/MXeczeMjiciIiIij4EKmIjB6pdzZ1kfPz5qVYX8DrbsOXuDFyZtYtCivVyPv2V0PBERERHJQipgItmAnY2Zt/zKsjbUnxdrlsRigZ+3naXxmDBmbjlNqpYlioiIiOQKKmAi2UhRF0fGtavBgl71qFy8ANE3k/l4yX5emLSRnaf/NjqeiIiIiDwiFTCRbKi2lxu/hzRkWOtqFHC0Zf/5GF6a8hcDFuzhalyS0fFERERE5CGpgIlkU7Y2ZjrV92LdgABerVUKgIU7z9F4TBjTNp0kJTXN4IQiIiIiYi0VMJFszj2/A6Nf9mXRO/WpXtKF2MQUhv12kOe/3sjWE9eMjiciIiIiVlABE8khni5diF/fbchnbavj6mTH4YuxtPtuC33n7uZyTKLR8UREREQkE1TARHIQG7OJ9nXLsC40gNfrlMZkgiURF2gyNpwfNpwgWcsSRURERLI1FTCRHKiQsz0jX3yKX99tQA1PV+KSUhjxxyFafrmBv45fNTqeiIiIiNyDCphIDuZTypVFb9dn9Es+uDnbc/RyHG98v5V35+wiKvqm0fFERERE5F9UwERyOLPZxKu1PVkXGkCnemUwm+CPvVE0GRPO5LBj3ErRskQRERGR7EIFTCSXKOhkx7AXqvNbSENqlSnEzeRURi+P5LkJ61l/5IrR8UREREQEFTCRXKeaR0EW9KrHuFd9cc/vwImr8XT8cRs9Z+7g3N8JRscTERERydNUwERyIZPJxItPl2LtAH+6NvDGxmxixYFLBI4L5+s1R0lKTjU6ooiIiEiepAImkou5ONoxJLgqy3r7UdfbjcTkNMauOkLLiX9x4G+T0fFERERE8hwVMJE8oFLxAszt8SxfvV6TYi4OnLl+k+8O29Bj1i7OXNOyRBEREZEnRQVMJI8wmUy09vVgTWgAbzX0wmyysC7yKoHjwxm36gg3b2lZooiIiMjjpgImksfkd7DlPy0q8oFvKvXLuXErJY2v1hwlcFw4Kw5cxGKxGB1RREREJNdSARPJo4rlg+mdnmFK+6fxKOjI+Rs36TlzJ52nbefElTij44mIiIjkSipgInmYyWQi6KkSrA71573G5bG3MRN+5AotJqxn1PLDJNxKMTqiiIiISK6iAiYiONnbMqBFJVb0a0RApSIkp1qYEnacpmPD+WNvlJYlioiIiGQRFTARSeft7sy0zrX5rsMzlCqUj6joRN6ds4s3p27l2OVYo+OJiIiI5HgqYCKSgclkonm14qzu70+fphWwtzWz6dg1npuwgf8uO0RckpYlioiIiDwsFTARuStHOxv6NavI6n7+BFYpRkqahe/Wn6DJmDB+jTivZYkiIiIiD0EFTETuq3RhJ37oVItpnWtTprATl2OT6DM3gnbfbeHwxRij44mIiIjkKCpgIpIpjSsXZUXfRgxoXhFHOzPbTl6n1VcbGfbbAaJvJhsdT0RERCRHUAETkUxztLPhvSYVWBMaQFD14qSmWZi26RRNx4axcOc50tK0LFFERETkflTARMRqJV3zMeXNZ5jZrQ5lizhzNe4WAxbs4ZVvN7P/fLTR8URERESyLRUwEXlofhWKsLxPIz4IqoyTvQ07T/9N64kb+XjJfm4k3DI6noiIiEi2owImIo/E3tZML/9yrA0NINjXgzQLzNxymiZjw5m77YyWJYqIiIj8DxUwEckSxQs68vXrNZnTvS4ViubnevwtPli0j7aTN7Hn7A2j44mIiIhkCypgIpKl6pdzZ1kfPz5qVYX8DrbsORdNm8mbGLRoL9fjtSxRRERE8jYVMBHJcnY2Zt7yK8vaAf68WLMkFgv8vO0sjceEMXPLaVK1LFFERETyKEML2Pr16wkODsbDwwOTycSSJUse+JnZs2fj6+uLk5MTJUqUoGvXrly7du2uY+fOnYvJZKJNmzb33F+vXr0wmUxMmDDh4U5CRO6paAFHxrWrwYJe9ahcvADRN5P5eMl+Wk/cyM7TfxsdT0REROSJM7SAxcfH4+vry6RJkzI1ftOmTXTs2JFu3bpx4MABFixYwLZt2+jevfsdY0+dOsWAAQPw8/O75/4WL17Mli1b8PDweOhzEJEHq+3lxu8hDRnWuhoFHG05cCGGl6b8xYAFe7gSm2R0PBEREZEnxtbIgwcFBREUFJTp8Zs3b8bLy4vevXsD4O3tTc+ePRk1alSGcampqbRv355hw4axYcMGbty4cce+zp8/T0hICCtWrKBVq1YPPHZSUhJJSf/3D8WYmBgAkpOTSU5OzvQ5PA63j290Dsk5jJozb9QuSYsq7oxZdYyFu86zcOc5Vhy4SJ8m5WhfxxNbG62Kzq7094xYS3NGrKU5I9bKTnPGmgyGFjBr1atXj8GDB7Ns2TKCgoK4fPkyCxcupGXLlhnGDR8+nKJFi9KtWzc2bNhwx37S0tLo0KEDAwcOpFq1apk69siRIxk2bNgd21euXImTk9PDnVAWW7VqldERJIcxas74OYBndVhw0oZz8SmMWBbJ1LDDvOydSnkXQyJJJunvGbGW5oxYS3NGrJUd5kxCQkKmx+aoAtagQQNmz55Nu3btSExMJCUlheDg4AxLGDdu3MjUqVOJiIi4535GjRqFra1t+pW0zBg0aBD9+/dPfx0TE4OnpyfNmzfHxcXYfzEmJyezatUqmjVrhp2dnaFZJGfILnOmZ5qF+TvPMW7VMaISkvn6gC2tfUrwfosKFHNxNCyX3Cm7zBnJOTRnxFqaM2Kt7DRnbq+Oy4wcVcAOHjxInz59GDJkCC1atCAqKoqBAwfSq1cvpk6dSmxsLB06dOD777/H3d39rvvYuXMnX375Jbt27cJkMmX62A4ODjg4ONyx3c7OzvBf+G3ZKYvkDEbPGTugY/2yBPuW4ouVkfy87QxL90ax5vBl+gZWpHMDL+y0LDFbMXrOSM6jOSPW0pwRa2WHOWPN8XNUARs5ciQNGjRg4MCBAPj4+ODs7Iyfnx8jRozg0qVLnDp1iuDg4PTPpKWlAWBra0tkZCQbNmzg8uXLlC5dOn1MamoqoaGhTJgwgVOnTj3RcxIRKORsz3/bPsVrtT0Z8usBIs7e4LNlh5i34yzDW1ejfvm7/wcVERERkZwmRxWwhIQEbG0zRraxsQHAYrFQuXJl9u3bl+H9jz76iNjYWL788ks8PT3p0KEDgYGBGca0aNGCDh060KVLl8d7AiJyXz6lXFn0dn0W7jzH58sPc+xyHG/8sJVWPiX4qFUVShTMZ3REERERkUdiaAGLi4vj2LFj6a9PnjxJREQEbm5ulC5dmkGDBnH+/HlmzJgBQHBwMN27d2fKlCnpSxD79u1LnTp10h8lX7169QzHcHV1zbC9cOHCFC5cOMMYOzs7ihcvTqVKlR7XqYpIJpnNJl6t7UmLasUZtyqSmVtO88feKNYeukxI0/J0a+iNg62N0TFFREREHoqhN1fs2LGDmjVrUrNmTQD69+9PzZo1GTJkCABRUVGcOXMmfXznzp0ZN24cEydOpHr16rzyyitUqlSJRYsWGZJfRB6fgk52DHuhOr+FNKRWmULcTE5l9PJIgiZsIPzIFaPjiYiIiDwUQ6+ABQQEYLFY7vn+9OnT79gWEhJCSEhIpo9xt338m+77Esm+qnkUZEGveizefZ7/LjvMiavxdPpxGy2qFeOjVlXxdMseXwMhIiIikhl6vJiIZHsmk4kXny7F2gH+dGvojY3ZxIoDlwgcF85Xa46SmJxqdEQRERGRTFEBE5Ecw8XRjo+fr8qy3n7U9XYjKSWNcauO0Hz8etYcumR0PBEREZEHUgETkRynUvECzO3xLF+9XpNiLg6cuZ5At5920G36dk5fizc6noiIiMg9qYCJSI5kMplo7evBmtAAevqXxdZsYs3hyzQbv55xKyO5eUvLEkVERCT7UQETkRwtv4Mtg4KqsLxvIxqWd+dWShpfrT1G4LhwVhy4eN8H/YiIiIg8aSpgIpIrlC+an5nd6jCl/dN4FHTk/I2b9Jy5k07TtnPiSpzR8UREREQAFTARyUVMJhNBT5Vgdag/7zUuj72NmfVHrtBiwnpGLT9Mwq0UoyOKiIhIHqcCJiK5jpO9LQNaVGJFv0YEVCpCcqqFKWHHaTo2nN/3XtCyRBERETGMCpiI5Fre7s5M61yb7zvWolShfERFJ/LenN28OXUrRy/FGh1PRERE8iAVMBHJ1UwmE82qFmN1f3/6NK2Ava2ZTceuEfTlBj774yBxSVqWKCIiIk+OCpiI5AmOdjb0a1aR1f38CaxSjJQ0C99vOEmTMWH8GnFeyxJFRETkiVABE5E8pXRhJ37oVItpnWvjVdiJy7FJ9JkbQbvvtnD4YozR8URERCSXUwETkTypceWiLO/biIEtKuFoZ2bbyeu0+mojQ5ceIPpmstHxREREJJdSARORPMvRzoZ3G5dnTWgAQdWLk5pmYfpfp2g6NoyFO8+RlqZliSIiIpK1VMBEJM8r6ZqPKW8+w8xudShbxJmrcbcYsGAPL3/zF/vPRxsdT0RERHIRFTARkf/Pr0IRlvdpxKCgyjjZ27DrzA1aT9zIR0v2cSPhltHxREREJBdQARMR+R/2tmZ6+pdjbWgAwb4epFlg1pYzNB4Txs/bzmhZooiIiDwSFTARkbsoXtCRr1+vyc/dn6Visfz8nZDMoEX7aDt5E3vO3jA6noiIiORQKmAiIvdRr1xh/ujtx0etqpDfwZY956JpM3kTH/yyl+vxWpYoIiIi1lEBExF5ADsbM2/5lWXtAH9erFkSiwXmbj9L4zFhzNxymlQtSxQREZFMUgETEcmkogUcGdeuBgt61aNKCReibybz8ZL9tJ64kZ2n/zY6noiIiOQAKmAiIlaq7eXGb+81YPgL1XBxtOXAhRhemvIXofP3cCU2yeh4IiIiko2pgImIPARbGzMd63mxdkAAr9YqBcAvu87RZGwY0zadJCU1zeCEIiIikh2pgImIPAL3/A6MftmXxe/U56mSBYlNTGHYbwd5/uuNbD1xzeh4IiIiks2ogImIZIGapQux5N0GfNa2Oq5Odhy+GEu777bQd+5uLsUkGh1PREREsgkVMBGRLGJjNtG+bhnWhQbwRt3SmEywJOICTcaE8f36EyRrWaKIiEiepwImIpLFCjnb89+2T/Hruw2o4elK/K1UPlt2iKAvN/DXsatGxxMREREDqYCJiDwmPqVcWfR2fUa/5IObsz3HLsfxxg9beXfOLi7cuGl0PBERETGACpiIyGNkNpt4tbYn60ID6FSvDGYT/LE3iqZjw5kcdoyklFSjI4qIiMgTpAImIvIEFHSyY9gL1fk9xI/aXoW4mZzK6OWRBE3YQPiRK0bHExERkSdEBUxE5Amq6uHC/J71GPeqL+75HThxNZ5OP26jx4wdnL2eYHQ8ERERecxUwEREnjCTycSLT5di3QB/ujX0xsZsYuXBSwSOC+erNUdJTNayRBERkdxKBUxExCAFHO34+PmqLOvtR11vN5JS0hi36gjNx69nzaFLRscTERGRx0AFTETEYJWKF2Buj2f56vWaFHNx4Mz1BLr9tINu07dz+lq80fFEREQkC6mAiYhkAyaTida+HqwNDaCnf1lszSbWHL5Ms/HrGbcykpu3tCxRREQkN1ABExHJRpwdbBkUVIXlfRvRsLw7t1LS+GrtMQLHhbN8/0UsFovREUVEROQRqICJiGRD5YvmZ2a3Okxp/zQeBR05f+MmvWbtpNO07Zy4Emd0PBEREXlIKmAiItmUyWQi6KkSrA71573G5bG3MbP+yBVaTFjPqOWHSbiVYnREERERsZIKmIhINudkb8uAFpVY0a8RAZWKkJxqYUrYcZqODef3vRe0LFFERCQHUQETEckhvN2dmda5Nt93rIWnWz6iohN5b85u2v+wlaOXYo2OJyIiIpmgAiYikoOYTCaaVS3Gqn7+9A2sgIOtmb+OXyPoyw189sdBYhOTjY4oIiIi96ECJiKSAzna2dA3sCKr+/sTWKUYKWkWvt9wkqZjw1my+7yWJYqIiGRTKmAiIjmYp5sTP3SqxbTOtfEq7MTl2CT6zoug3XdbOHwxxuh4IiIi8i8qYCIiuUDjykVZ0a8RA1tUwtHOzLaT12n11UaGLj1A9E0tSxQREckuVMBERHIJB1sb3m1cnjWhAbR8qjipaRam/3WKpmPDWLDjLGlpWpYoIiJiNBUwEZFcpqRrPia3f4aZ3epQtogzV+NuMXDhXl7+5i/2n482Op6IiEiepgImIpJL+VUowvI+jRgUVBknext2nblB8MSNfLRkHzcSbhkdT0REJE9SARMRycXsbc309C/H2tAAWvt6YLHArC1naDwmjJ+3ndGyRBERkSdMBUxEJA8oXtCRr16vyc/dn6Visfz8nZDMoEX7aDt5ExFnbxgdT0REJM8wtICtX7+e4OBgPDw8MJlMLFmy5IGfmT17Nr6+vjg5OVGiRAm6du3KtWvX7jp27ty5mEwm2rRpk74tOTmZ//znPzz11FM4Ozvj4eFBx44duXDhQhadlYhI9lWvXGH+6O3Hx89XpYCDLXvORdN28iY++GUv1+O1LFFERORxM7SAxcfH4+vry6RJkzI1ftOmTXTs2JFu3bpx4MABFixYwLZt2+jevfsdY0+dOsWAAQPw8/PLsD0hIYFdu3bx8ccfs2vXLhYtWkRkZCStW7fOknMSEcnu7GzMdGvozZoB/rxYsyQWC8zdfpbGY8KYufkUqVqWKCIi8tjYGnnwoKAggoKCMj1+8+bNeHl50bt3bwC8vb3p2bMno0aNyjAuNTWV9u3bM2zYMDZs2MCNGzfS3ytYsCCrVq3KMH7ixInUqVOHM2fOULp06Yc/IRGRHKRoAUfGtavB63VLM+TXAxyKiuHjXw8wd/tZhr9QnWfKFDI6ooiISK5jaAGzVr169Rg8eDDLli0jKCiIy5cvs3DhQlq2bJlh3PDhwylatCjdunVjw4YND9xvdHQ0JpMJV1fXe45JSkoiKSkp/XVMTAzwz5LG5GRjv+T09vGNziE5h+aM/K8aJQuwqGcd5u44x/jVxzhwIYaXpvxF25oevN+8Au75HTRnxGqaM2ItzRmxVnaaM9ZkMFkslmyx1sRkMrF48eIM92vdzYIFC+jatSuJiYmkpKQQHBzML7/8gp2dHQAbN27ktddeIyIiAnd3dzp37syNGzfueX9ZYmIiDRo0oHLlysyePfuexx06dCjDhg27Y/ucOXNwcnLK9HmKiGRnscnw+xkzWy7/s0Ld0cZCkGcafsUt2JgMDiciIpJNJSQk8MYbbxAdHY2Li8t9x+aoAnbw4EECAwPp168fLVq0ICoqioEDB1K7dm2mTp1KbGwsPj4+TJ48OX1p4/0KWHJyMi+99BLnzp0jLCzsvj+su10B8/T05OrVqw/8IT9uycnJrFq1imbNmqUXUZH70ZyRB4k4e4Nhvx9m/4V/rvZXLOpMiyLRvP2S5oxkjv6eEWtpzoi1stOciYmJwd3dPVMFLEctQRw5ciQNGjRg4MCBAPj4+ODs7Iyfnx8jRozg0qVLnDp1iuDg4PTPpKWlAWBra0tkZCTlypUD/vmFvfrqq5w+fZq1a9c+8Afl4OCAg4PDHdvt7OwM/4Xflp2ySM6gOSP3UrtsEX59z52528/wxYpIjlyO58hlW07ZHOKj56tRzMXR6IiSQ+jvGbGW5oxYKzvMGWuOn6MKWEJCAra2GSPb2NgAYLFYqFy5Mvv27cvw/kcffURsbCxffvklnp6ewP+Vr6NHj7Ju3ToKFy78ZE5ARCQHsTGbaF+3DC2rl2DU8kPM236W3/ZeZO3hK/QJrECXBt7Y2ejrJEVERKxhaAGLi4vj2LFj6a9PnjxJREQEbm5ulC5dmkGDBnH+/HlmzJgBQHBwMN27d2fKlCnpSxD79u1LnTp18PDwAKB69eoZjnH7wRq3tycnJ/Pyyy+za9cufv/9d1JTU7l48SIAbm5u2NvbP+7TFhHJUQo52/Np66qUSjzFqr8Ls+dcNP9ddpj5O84xrHU1GpR3NzqiiIhIjmFoAduxYweNGzdOf92/f38AOnXqxPTp04mKiuLMmTPp73fu3JnY2FgmTpxIaGgorq6uNGnS5I7H0N/P+fPnWbp0KQA1atTI8N66desICAh4+BMSEcnFPPPD/Jfr8Ou+S4z68zDHLsfR/oettHqqBB+2qoKHaz6jI4qIiGR7hhawgIAA7vcMkOnTp9+xLSQkhJCQkEwf49/78PLyuu8xRUTk3sxmE6/W8qRF1eKMWxXJzC2n+WNfFGsPXyakaXm6NfTGwdbG6JgiIiLZlhbvi4iI1Qo62THsher8HuJHba9C3ExOZfTySJ6bsIHwI1eMjiciIpJtqYCJiMhDq+rhwvye9RjfzpciBRw4eTWeTj9uo8eMHZy9nmB0PBERkWxHBUxERB6JyWSibc1SrA31562G3tiYTaw8eInAceF8ufooicmpRkcUERHJNlTAREQkSxRwtOOj56vyZx8/ni3rRlJKGuNXH6H5+PWsOXTJ6HgiIiLZggqYiIhkqYrFCvBz92f56vWaFHNx4Mz1BLr9tIOu07dz+lq80fFEREQMpQImIiJZzmQy0drXg7WhAfT0L4udjYm1hy/TbPx6xq2M5OYtLUsUEZG8yeoCdvbsWc6dO5f+etu2bfTt25fvvvsuS4OJiEjO5+xgy6CgKvzZpxF+Fdy5lZLGV2uPETgunOX7L+prQUREJM+xuoC98cYbrFu3DoCLFy/SrFkztm3bxocffsjw4cOzPKCIiOR85YvmZ0bXOnzz5tOUdM3H+Rs36TVrJx1/3MaJK3FGxxMREXlirC5g+/fvp06dOgDMnz+f6tWr89dffzF79uy7fnGyiIgI/LMs8bnqJVjd35/3GpfH3sbMhqNXaTFhPaOWHyY+KcXoiCIiIo+d1QUsOTkZBwcHAFavXk3r1q0BqFy5MlFRUVmbTkREcp189jYMaFGJlf0a0bhSEZJTLUwJO07guHB+33tByxJFRCRXs7qAVatWjW+++YYNGzawatUqnnvuOQAuXLhA4cKFszygiIjkTl7uzvzYuTY/dKyFp1s+oqITeW/Obtr/sJWjl2KNjiciIvJYWF3ARo0axbfffktAQACvv/46vr6+ACxdujR9aaKIiEhmmEwmAqsWY1U/f/oGVsDB1sxfx68R9OUGRvx+kNjEZKMjioiIZClbaz8QEBDA1atXiYmJoVChQunbe/TogZOTU5aGExGRvMHRzoa+gRV56elSDP/9IKsOXuKHjSdZuucCg1tW4YUaHphMJqNjioiIPDKrr4DdvHmTpKSk9PJ1+vRpJkyYQGRkJEWLFs3ygCIiknd4ujnxfcdaTOtSG6/CTlyOTaLvvAjafbuFQ1ExRscTERF5ZFYXsBdeeIEZM2YAcOPGDerWrcvYsWNp06YNU6ZMyfKAIiKS9zSuVJQV/RoxsEUlHO3MbDt1nee/3sjQpQeIvqlliSIiknNZXcB27dqFn58fAAsXLqRYsWKcPn2aGTNm8NVXX2V5QBERyZscbG14t3F51oQG0PKp4qSmWZj+1ymajg1jwY6zpKXpaYkiIpLzWF3AEhISKFCgAAArV67kxRdfxGw28+yzz3L69OksDygiInlbSdd8TG7/DDO71aFcEWeuxt1i4MK9vPzNX+w/H210PBEREatYXcDKly/PkiVLOHv2LCtWrKB58+YAXL58GRcXlywPKCIiAuBXoQh/9mnEoKDKONnbsOvMDYInbuSjJfu4kXDL6HgiIiKZYnUBGzJkCAMGDMDLy4s6depQr1494J+rYTVr1szygCIiIrfZ25rp6V+OtaEBtPb1wGKBWVvO0HhMGD9vO6NliSIiku1ZXcBefvllzpw5w44dO1ixYkX69qZNmzJ+/PgsDSciInI3xQs68tXrNfm5+7NULJafvxOSGbRoH20nbyLi7A2j44mIiNyT1QUMoHjx4tSsWZMLFy5w7tw5AOrUqUPlypWzNJyIiMj91CtXmD96+/Hx81Up4GDLnnPRtJ28iQ9+2cu1uCSj44mIiNzB6gKWlpbG8OHDKViwIGXKlKFMmTK4urry6aefkpaW9jgyioiI3JOdjZluDb1ZM8CfF58uicUCc7efpcnYcGZuPkWqliWKiEg2YnUB+/DDD5k4cSKff/45u3fvZvfu3fz3v//l66+/5uOPP34cGUVERB6oaAFHxr1ag4W96lGlhAvRN5P5+NcDBH+9kZ2nrxsdT0REBABbaz/w008/8cMPP9C6dev0bT4+PpQsWZJ33nmHzz77LEsDioiIWKOWlxu/vdeAOdvOMGZFJAejYnhpymZeeroUHwRVpkgBB6MjiohIHmb1FbDr16/f9V6vypUrc/26/gujiIgYz9bGTMd6XqwbEEC7Wp4A/LLrHE3GhPHjxpOkpGrJvIiIGMPqAubr68vEiRPv2D5x4kR8fX2zJJSIiEhWKJzfgVEv+7D4nfo8VbIgsUkpDP/9IK2+2sjWE9eMjiciInmQ1UsQR48eTatWrVi9enX6d4Bt3ryZs2fPsmzZsiwPKCIi8qhqli7EkncbMG/7WUavOEzkpVjafbeFF2p4MLhlFYq5OBodUURE8girr4D5+/tz5MgR2rZty40bN7hx4wYvvvgikZGR+Pn5PY6MIiIij8zGbOKNuqVZFxpA+7qlMZng14gLNBkTxnfrj5OsZYkiIvIEWH0FDMDDw+OOh22cO3eOHj168N1332VJMBERkcehkLM9n7V9itdql+bjX/cTcfYG/112mPk7zjGsdTUalHc3OqKIiORiD/VFzHdz7do1pk6dmlW7ExEReayeKlWQRW/XZ/TLPhR2tufY5Tja/7CVd2fv4sKNm0bHExGRXCrLCpiIiEhOYzabeLWWJ2tDA+hc3wuzCf7YF0XTseFMWneMpJRUoyOKiEguowImIiJ5XkEnO4a2rsbvIX7U9irEzeRUvlgRyXMTNhAWednoeCIikouogImIiPx/VT1cmN+zHuPb+VKkgAMnr8bTedp2eszYwdnrCUbHExGRXCDTD+F48cUX7/v+jRs3HjWLiIiI4UwmE21rliKwSjG+XH2UaX+dYuXBS4QfucI7AeXp6V8WRzsbo2OKiEgOlekCVrBgwQe+37Fjx0cOJCIikh0UcLTjo+er8mptT4b8up8tJ64zfvURftl1jiHPVyWwajGjI4qISA6U6QI2bdq0x5lDREQkW6pYrAA/d3+W3/dG8dkfhzhzPYG3ZuygSeWifBJclTKFnY2OKCIiOYjuARMREXkAk8lEsK8Ha0L96elfFjsbE2sPX6bZuPWMXRnJzVt6WqKIiGSOCpiIiEgmOTvYMiioCn/2aYRfBXdupabx9dpjBI4LZ/n+KCwWi9ERRUQkm1MBExERsVL5ovmZ0bUO37z5NCVd83H+xk16zdpFxx+3cfxKnNHxREQkG1MBExEReQgmk4nnqpdgdX9/QpqUx97GzIajV3luwno+//Mw8UkpRkcUEZFsSAVMRETkEeSztyG0eSVW9mtE40pFSE618E34cQLHhfP73gtaligiIhlk6imIS5cuzfQOW7du/dBhREREciovd2d+7FybNYcuM+z3A5y9fpP35uxmTrkzDGtdjQrFChgdUUREsoFMFbA2bdpkamcmk4nUVD0JSkRE8iaTyURg1WI0rODOt+EnmBx2jL+OXyPoyw10ru9Fn8AKFHC0MzqmiIgYKFNLENPS0jL1R+VLREQEHO1s6BNYgdX9/WlWtRgpaRZ+2HiSJmPDWbz7nJYliojkYboHTERE5DHxdHPi+461mNalNl6FnbgSm0S/eXto9+0WDkXFGB1PREQMkKkliP8WHx9PeHg4Z86c4datWxne6927d5YEExERyS0aVypK/X6F+WHDSb5ee5Rtp67T6qsNdKznRb9mFSmYT8sSRUTyCqsL2O7du2nZsiUJCQnEx8fj5ubG1atXcXJyomjRoipgIiIid+Fga8O7jcvTpmZJPvvjIMv2XWT6X6f4bc8F/hNUmZefLoXZbDI6poiIPGZWL0Hs168fwcHB/P333+TLl48tW7Zw+vRpnnnmGcaMGfM4MoqIiOQaJV3zMbn9M8zqVpdyRZy5Fn+L9xfu5aVv/mL/+Wij44mIyGNmdQGLiIggNDQUs9mMjY0NSUlJeHp6Mnr0aAYPHvw4MoqIiOQ6DSu482efRgxuWRlnext2n7lB8MSNfLh4HzcSbj14ByIikiNZXcDs7Owwm//5WNGiRTlz5gwABQsW5OzZs1bta/369QQHB+Ph4YHJZGLJkiUP/Mzs2bPx9fXFycmJEiVK0LVrV65du3bXsXPnzsVkMt3xGH2LxcKQIUMoUaIE+fLlIzAwkKNHj1qVXURE5FHZ25rp0agca0IDaO3rgcUCs7eeofGYMH7edobUND0tUUQkt7G6gNWsWZPt27cD4O/vz5AhQ5g9ezZ9+/alevXqVu0rPj4eX19fJk2alKnxmzZtomPHjnTr1o0DBw6wYMECtm3bRvfu3e8Ye+rUKQYMGICfn98d740ePZqvvvqKb775hq1bt+Ls7EyLFi1ITEy0Kr+IiEhWKF7Qka9er8nP3Z+lYrH8/J2QzKBF+2g7eRMRZ28YHU9ERLKQ1QXsv//9LyVKlADgs88+o1ChQrz99ttcuXKFb7/91qp9BQUFMWLECNq2bZup8Zs3b8bLy4vevXvj7e1Nw4YN6dmzJ9u2bcswLjU1lfbt2zNs2DDKli2b4T2LxcKECRP46KOPeOGFF/Dx8WHGjBlcuHAhU1fgREREHpd65QrzR28/Pn6+KgUcbNl7Lpq2kzfxwS97uRaXZHQ8ERHJAlY/BbFWrVrp/3fRokVZvnx5lga6n3r16jF48GCWLVtGUFAQly9fZuHChbRs2TLDuOHDh1O0aFG6devGhg0bMrx38uRJLl68SGBgYPq2ggULUrduXTZv3sxrr71212MnJSWRlPR//+MXE/PP97ckJyeTnJycVaf4UG4f3+gcknNozoi1NGeerI51SxFUtQhfrDzC4ogo5m4/y7J9UfQLLM/rtT2xyQFPS9ScEWtpzoi1stOcsSaD1QWsSZMmLFq0CFdX1wzbY2JiaNOmDWvXrrV2l5nWoEEDZs+eTbt27UhMTCQlJYXg4OAMSxg3btzI1KlTiYiIuOs+Ll68CECxYsUybC9WrFj6e3czcuRIhg0bdsf2lStX4uTk9BBnk/VWrVpldATJYTRnxFqaM09WQD4oXQ0WnrThfEIKw34/zA9rD/FK2VS8CxidLnM0Z8RamjNirewwZxISEjI91uoCFhYWdseXLwMkJibecbUpqx08eJA+ffowZMgQWrRoQVRUFAMHDqRXr15MnTqV2NhYOnTowPfff4+7u3uWHnvQoEH0798//XVMTAyenp40b94cFxeXLD2WtZKTk1m1ahXNmjXDzk5f5ikPpjkj1tKcMdbbaRZ+3n6W8auPcT4hhQn7bWlbowTvt6iIe34Ho+PdleaMWEtzRqyVnebM7dVxmZHpArZ37970//vgwYMZrhalpqayfPlySpYsmekDP4yRI0fSoEEDBg4cCICPjw/Ozs74+fkxYsQILl26xKlTpwgODk7/TFpaGgC2trZERkZSvHhxAC5dupR+L9vt1zVq1LjnsR0cHHBwuPN/5Ozs7Az/hd+WnbJIzqA5I9bSnDGGHdClYTla1yjF6OWRzNtxlsURUaw+dIV+zSrSsV4ZbG2svq37idCcEWtpzoi1ssOcseb4mS5gNWrUwGQyYTKZaNKkyR3v58uXj6+//jrTB34YCQkJ2NpmjGxjYwP883CNypUrs2/fvgzvf/TRR8TGxvLll1/i6emJnZ0dxYsXZ82aNemFKyYmhq1bt/L2228/1vwiIiKPonB+B0a97MPrdUsz5Nf97D0XzfDfDzJv+1mGvVCNZ8sWNjqiiIg8QKYL2MmTJ7FYLJQtW5Zt27ZRpEiR9Pfs7e0pWrRoehnKrLi4OI4dO5bhGBEREbi5uVG6dGkGDRrE+fPnmTFjBgDBwcF0796dKVOmpC9B7Nu3L3Xq1MHDwwPgjkfh375X7X+39+3blxEjRlChQgW8vb35+OOP8fDwuOP7wkRERLKjGp6uLH6nAfO2n2X0isNEXorlte+20NrXgw9bVaGYi6PREUVE5B4yXcDKlCkD/N+SvqywY8cOGjdunP769j1WnTp1Yvr06URFRaV/0TNA586diY2NZeLEiYSGhuLq6kqTJk0YNWqUVcd9//33iY+Pp0ePHty4cYOGDRuyfPlyHB31P1giIpIz2JhNvFG3NEHVizNmZSRztp1h6Z4LrDl0iT6BFehc3xt72+y5LFFEJC+z+iEcAMePH2fChAkcOnQIgKpVq9KnTx/KlStn1X4CAgKwWCz3fH/69Ol3bAsJCSEkJCTTx7jbPkwmE8OHD2f48OGZ3o+IiEh2VMjZns/aPsVrtUvz8a/7iTh7g/8uO8y87WcZ/kJ1GpTP2odSiYjIo7H6P42tWLGCqlWrsm3bNnx8fPDx8WHr1q1Uq1YtWzwCUkREJC96qlRBFr1dn9Ev+1DY2Z7jV+Jp/8NW3p29iws3bhodT0RE/j+rr4B98MEH9OvXj88///yO7f/5z39o1qxZloUTERGRzDObTbxay5MW1YozftURZmw+xR/7olh7+DLvNSnPW37eONhad7+2iIhkLauvgB06dIhu3brdsb1r164cPHgwS0KJiIjIwyuYz46hravxe4gftb0KcTM5lS9WRPLchA2ERV42Op6ISJ5mdQErUqQIERERd2yPiIigaNGiWZFJREREskBVDxfm96zH+Ha+FCngwMmr8XSetp3uM3Zw9nqC0fFERPKkTC9BHD58OAMGDKB79+706NGDEydOUL9+fQA2bdrEqFGj0p9iKCIiItmDyWSibc1SBFYpxperjzLtr1OsOniJ9Ueu8E5AeXr6l8XRTssSRUSelEwXsGHDhtGrVy8+/vhjChQowNixYxk0aBAAHh4eDB06lN69ez+2oCIiIvLwCjja8dHzVXm1tief/HqAzSeuMX71EX7ZdY4hz1clsGoxoyOKiOQJmV6CePtx8SaTiX79+nHu3Dmio6OJjo7m3Llz9OnTB5PJ9NiCioiIyKOrWKwAc7rX5evXa1LcxZEz1xN4a8YOuk7fzqmr8UbHExHJ9ay6B+zfBatAgQIUKFAgSwOJiIjI42UymQj29WBNqD+9/MthZ2Ni7eHLNB+/nrErI7l5K9XoiCIiuZZVj6GvWLHiA69yXb9+/ZECiYiIyJPh7GDLB0GVeaVWKYYuPcCGo1f5eu0xFu06z8fPV6FFteJa3SIiksWsKmDDhg2jYMGCjyuLiIiIGKBckfzM6FqHFQcu8unvhzh/4ya9Zu3Cr4I7Q1tXo1yR/EZHFBHJNawqYK+99poeNS8iIpILmUwmnqteAv+KRZkcdoxvw0+w4ehVnpuwnm4NyxLSpDzODlb9s0FERO4i0/eAaQmCiIhI7pfP3obQ5pVY2a8RjSsVITnVwjfhx2k6Npzf9lxIfyiXiIg8HKufgigiIiK5n5e7M9O61OGHjrXwdMvHxZhEQn7ezRvfb+XIpVij44mI5FiZLmBpaWlafigiIpLHBFYtxqp+/vQLrIiDrZnNJ67R8ssNjPj9ILGJyUbHExHJcax6DL2IiIjkPY52NvQJrMDq/v40r1qMlDQLP2w8SZOx4SzefU6rZERErKACJiIiIpni6ebEdx1rMb1LbbwKO3ElNol+8/bw6rebOXghxuh4IiI5ggqYiIiIWCWgUlFW9GvEwBaVyGdnw/ZTf/P81xsYuvQA0Te1LFFE5H5UwERERMRqDrY2vNu4PKtD/Wn5VHHSLDD9r1M0GRPG/B1nSUvTskQRkbtRARMREZGHVtI1H5PbP8OsbnUpV8SZa/G3eH/hXtr9sI2zcUanExHJflTARERE5JE1rODOn30aMbhlZZztbYg4G83YfTYMWXqQGwm3jI4nIpJtqICJiIhIlrC3NdOjUTnWhAYQ7FMcCyZ+3n6OxmPCmLP1DKlaligiogImIiIiWat4QUfGveJDSNUUKhbNz98JyQxevI+2kzcRcfaG0fFERAylAiYiIiKPRfmCsOSdZxnyfFUKONiy91w0bSZt4j8L93ItLsnoeCIihlABExERkcfGzsZM14berBngz0tPlwJg3o6zNB4TxozNp7QsUUTyHBUwEREReeyKFnBk7Ku+LOxVj6olXIhJTGHIrwcI/nojO05dNzqeiMgTowImIiIiT0wtLzd+C2nIpy9Uw8XRloNRMbz8zWb6z4/gcmyi0fFERB47FTARERF5omzMJjrU82LdgADa1fIEYNGu8zQdE87UjSdJSU0zOKGIyOOjAiYiIiKGKJzfgVEv+7Dk3Qb4lCpIbFIKn/5+kFZfbWTLiWtGxxMReSxUwERERMRQNTxdWfxOA0a++BSFnOyIvBTLa99toffPu7kUo2WJIpK7qICJiIiI4WzMJl6vU5q1oQG0r1sakwmW7rlAkzFhfBt+nFspWpYoIrmDCpiIiIhkG4Wc7fms7VP89l5DapZ2Jf5WKiP/PEzQl+vZePSq0fFERB6ZCpiIiIhkO9VLFuSXXvUZ/bIPhZ3tOX4lnjenbuWd2Tu5cOOm0fFERB6aCpiIiIhkS2aziVdrebJ2QACd63thNsGyfRdpOjacSeuOkZSSanREERGrqYCJiIhItlYwnx1DW1fjj95+1PYqxM3kVL5YEclzEzYQFnnZ6HgiIlZRARMREZEcoUoJF+b3rMeEdjUoUsCBk1fj6TxtO91n7ODs9QSj44mIZIoKmIiIiOQYJpOJNjVLsjbUn7caemNjNrHq4CUCx4UzYfUREpO1LFFEsjcVMBEREclxCjja8dHzVfmzjx/1yhYmKSWNCauP0mx8OKsPXjI6nojIPamAiYiISI5VsVgB5nSvy8Q3alLcxZGz12/y1owddJ2+nVNX442OJyJyBxUwERERydFMJhPP+3iwJtSfXv7lsLMxsfbwZZqPX8+YFZHcvKVliSKSfaiAiYiISK7g7GDLB0GVWd63EX4V3LmVmsbEdccIHBfO8v1RWCwWoyOKiKiAiYiISO5Srkh+ZnStwzdvPk1J13ycv3GTXrN20fHHbRy/Emd0PBHJ41TAREREJNcxmUw8V70Eq/v7E9KkPPY2ZjYcvcpzE9bz+Z+HiU9KMTqiiORRKmAiIiKSa+WztyG0eSVW9mtEk8pFSU618E34cZqODee3PRe0LFFEnjgVMBEREcn1vNyd+bFzbX7oWAtPt3xcjEkk5OfdvPH9Vo5cijU6nojkISpgIiIikmcEVi3Gqn7+9AusiIOtmc0nrhH05QY+/f0gsYnJRscTkTxABUxERETyFEc7G/oEVmB1f3+aVy1GapqFqRtP0mRsOIt3n9OyRBF5rFTAREREJE/ydHPiu461mN6lNt7uzlyJTaLfvD28+u1mDl6IMTqeiORSKmAiIiKSpwVUKsryvn4MbFGJfHY2bD/1N89/vYGhSw8QfVPLEkUka6mAiYiISJ7nYGvDu43LszrUn1ZPlSDNAtP/OkWTMWHM33GWtDQtSxSRrGFoAVu/fj3BwcF4eHhgMplYsmTJAz8ze/ZsfH19cXJyokSJEnTt2pVr166lv79o0SJq1aqFq6srzs7O1KhRg5kzZ2bYR1xcHO+99x6lSpUiX758VK1alW+++SarT09ERERymJKu+ZjU/mlmdatLuSLOXIu/xfsL9/LSN3+x71y00fFEJBcwtIDFx8fj6+vLpEmTMjV+06ZNdOzYkW7dunHgwAEWLFjAtm3b6N69e/oYNzc3PvzwQzZv3szevXvp0qULXbp0YcWKFelj+vfvz/Lly5k1axaHDh2ib9++vPfeeyxdujTLz1FERERynoYV3PmzTyMGt6yMs70Nu8/coPWkjXy4eB9/x98yOp6I5GCGFrCgoCBGjBhB27ZtMzV+8+bNeHl50bt3b7y9vWnYsCE9e/Zk27Zt6WMCAgJo27YtVapUoVy5cvTp0wcfHx82btyYPuavv/6iU6dOBAQE4OXlRY8ePfD19c2wHxEREcnb7G3N9GhUjrUDAnihhgcWC8zeeobGY8OYs/UMqVqWKCIPwdboANaoV68egwcPZtmyZQQFBXH58mUWLlxIy5Yt7zreYrGwdu1aIiMjGTVqVPr2+vXrs3TpUrp27YqHhwdhYWEcOXKE8ePH3/PYSUlJJCUlpb+Oifnn6UjJyckkJxt7g+7t4xudQ3IOzRmxluaMWCs3zRm3fDaMeak6rzztwfDfD3PkchyDF+/j522nGdKqMjU8XY2OmCvkpjkjT0Z2mjPWZDBZssmXXZhMJhYvXkybNm3uO27BggV07dqVxMREUlJSCA4O5pdffsHOzi59THR0NCVLliQpKQkbGxsmT55M165d099PSkqiR48ezJgxA1tbW8xmM99//z0dO3a853GHDh3KsGHD7tg+Z84cnJycrD9hERERyXFS02DDJRN/njWTmGoC4NmiaQSXTiO/3QM+LCK5VkJCAm+88QbR0dG4uLjcd2yOKmAHDx4kMDCQfv360aJFC6Kiohg4cCC1a9dm6tSp6ePS0tI4ceIEcXFxrFmzhk8//ZQlS5YQEBAAwJgxY/j+++8ZM2YMZcqUYf369QwaNIjFixcTGBh412Pf7QqYp6cnV69efeAP+XFLTk5m1apVNGvWLEMRFbkXzRmxluaMWCu3z5krsUl8seooi3dfAMDF0ZZ+geV5vbYnNmaTwelyptw+ZyTrZac5ExMTg7u7e6YKWI5agjhy5EgaNGjAwIEDAfDx8cHZ2Rk/Pz9GjBhBiRIlADCbzZQvXx6AGjVqcOjQIUaOHElAQAA3b95k8ODBLF68mFatWqXvJyIigjFjxtyzgDk4OODg4HDHdjs7O8N/4bdlpyySM2jOiLU0Z8RauXXOeLjZMb5dTdrXLcOQXw9wMCqGYb8fZsHOCwx/oRq1vNyMjphj5dY5I49Pdpgz1hw/R30PWEJCAmZzxsg2NjbAP/d73UtaWlr61avb92zdbT9paWlZnFhERERys1pebvwW0pBPX6iGi6MtB6NiePmbzfSfH8Hl2ESj44lINmToFbC4uDiOHTuW/vrkyZNERETg5uZG6dKlGTRoEOfPn2fGjBkABAcH0717d6ZMmZK+BLFv377UqVMHDw8P4J+rZLVq1aJcuXIkJSWxbNkyZs6cyZQpUwBwcXHB39+fgQMHki9fPsqUKUN4eDgzZsxg3LhxT/6HICIiIjmajdlEh3petHyqBF+siGTejrMs2nWeVQcu0bdZRTrWK4OdTY76b94i8hgZWsB27NhB48aN01/3798fgE6dOjF9+nSioqI4c+ZM+vudO3cmNjaWiRMnEhoaiqurK02aNMnwhMP4+Hjeeecdzp07R758+ahcuTKzZs2iXbt26WPmzp3LoEGDaN++PdevX6dMmTJ89tln9OrV6wmctYiIiORGhfM78PlLPrxWpzRDft3P3nPRfPr7QeZvP8uwF6rxbNnCRkcUkWzA0AIWEBBw36WD06dPv2NbSEgIISEh9/zMiBEjGDFixH2PW7x4caZNm5bpnCIiIiKZVcPTlcXvNGD+jrOMXn6YyEuxvPbdFlr7ejC4ZRWKF3Q0OqKIGEjXw0VERESymI3ZxOt1SrNuQABvPlsakwmW7rlA07FhfBt+nFspuu9cJK9SARMRERF5TFyd7BnR5il+e68hNUu7En8rlZF/Hiboy/VsPHrV6HgiYgAVMBEREZHHrHrJgvzSqz5fvOxDYWd7jl+J582pW3ln9k7O37hpdDwReYJUwERERESeALPZxCu1PFk7IIDO9b0wm2DZvosEjg1n0rpjJKWkGh1RRJ4AFTARERGRJ6hgPjuGtq7GH739qOPlxs3kVL5YEUmL8etZF3nZ6Hgi8pipgImIiIgYoEoJF+b1fJYJ7WpQpIADp64l0GXadrrP2MHZ6wlGxxORx0QFTERERMQgJpOJNjVLsjbUn+5+3tiaTaw6eInAceFMWH2ExGQtSxTJbVTARERERAxWwNGOD1tV5c8+ftQrW5iklDQmrD5Ks/HhrDp46b7fmyoiOYsKmIiIiEg2UaFYAeZ0r8vEN2pS3MWRs9dv0n3GDrpO386pq/FGxxORLKACJiIiIpKNmEwmnvfxYE2oP28HlMPOxsS6yCs0H7+eMSsiuXlLyxJFcjIVMBEREZFsyNnBlv88V5nlfRvhV8GdW6lpTFx3jMBx4fy5L0rLEkVyKBUwERERkWysXJH8zOhah2/efIaSrvk4f+Mmb8/eRccft3H8SpzR8UTESipgIiIiItmcyWTiuerFWd3fn5Am5bG3MbPh6FWem7CekX8eIj4pxeiIIpJJKmAiIiIiOUQ+extCm1diZb9GNKlclORUC9+Gn6Dp2HCW7rmgZYkiOYAKmIiIiEgO4+XuzI+dazO1Uy1KuzlxMSaR3j/v5o3vt3LkUqzR8UTkPlTARERERHKoplWKsbJfI/o3q4iDrZnNJ64R9OUGPv39ILGJyUbHE5G7UAETERERycEc7Wzo3bQCq/v707xqMVLTLEzdeJLGY8JZtOucliWKZDMqYCIiIiK5gKebE991rMX0LrXxdnfmalwS/efv4dVvN3PwQozR8UTk/1MBExEREclFAioVZXlfPwa2qEQ+Oxu2n/qb57/ewCe/7if6ppYlihhNBUxEREQkl3GwteHdxuVZE+pPq6dKkGaBnzafpsmYMOZvP0tampYlihhFBUxEREQkl/Jwzcek9k8z+626lC+an2vxt3j/l7289M1f7DsXbXQ8kTxJBUxEREQkl2tQ3p1lvf0Y3LIyzvY27D5zg9aTNjJ48T7+jr9ldDyRPEUFTERERCQPsLc106NROdYOCOCFGh5YLDBn6xkajw1j9tbTpGpZosgToQImIiIikocUc3Hky9dqMq/Hs1QuXoAbCcl8uHg/bSZtYveZv42OJ5LrqYCJiIiI5EF1yxbm95CGDHm+KgUcbNl3Ppq2k//i/YV7uBaXZHQ8kVxLBUxEREQkj7K1MdO1oTdrBwTw0tOlAJi/4xyNx4QxY/MpUlLTDE4okvuogImIiIjkcUUKODD2VV9+ebseVUu4EJOYwpBfDxA8cRM7Tl03Op5IrqICJiIiIiIAPFPGjd9CGvLpC9VwcbTlUFQML3+zmf7zI7gcm2h0PJFcQQVMRERERNLZmE10qOfFugEBvFbbE5MJFu06T9Mx4UzdeJJkLUsUeSQqYCIiIiJyh8L5Hfj8JR8Wv9MAn1IFiU1K4dPfD/L8VxvZfPya0fFEciwVMBERERG5pxqerix5pwEjX3yKQk52RF6K5fXvtxDy824uRmtZooi1VMBERERE5L7MZhOv1ynNugEBvPlsaUwm+G3PBZqODePb8OPcStGyRJHMUgETERERkUxxdbJnRJun+O29htQs7Ur8rVRG/nmYoC/Xs/HoVaPjieQIKmAiIiIiYpXqJQvyS6/6fPGyD4Wd7Tl+JZ43p27l7Vk7OX/jptHxRLI1FTARERERsZrZbOKVWp6sHRBA5/pemE3w5/6LBI4NZ9K6YySlpBodUSRbUgETERERkYdWMJ8dQ1tX44/eftTxcuNmcipfrIikxfj1rIu8bHQ8kWxHBUxEREREHlmVEi7M6/ksE9rVoEgBB05dS6DLtO10n7GDs9cTjI4nkm2ogImIiIhIljCZTLSpWZK1of509/PG1mxi1cFLBI4LZ8LqIyQma1miiAqYiIiIiGSpAo52fNiqKn/28aN+ucIkpaQxYfVRmo0PZ9XBS1gsFqMjihhGBUxEREREHosKxQow+626THyjJsVdHDl7/SbdZ+ygy/TtnLoab3Q8EUOogImIiIjIY2MymXjex4M1of68HVAOOxsTYZFXaD5+PWNWRHLzlpYlSt6iAiYiIiIij52zgy3/ea4yy/s2wq+CO7dS05i47hiB48L5c1+UliVKnmFrdAARERERyTvKFcnPjK51WHHgEp/+fpDzN27y9uxdNChXGP8CRqcTefx0BUxEREREniiTycRz1Yuzur8/vZuUx97WzKbj1xi1x4bRK44Ql5RidESRx0YFTEREREQMkc/ehv7NK7GqXyMCKrqTajHx/cZTNB0bxtI9F7QsUXIlFTARERERMVSZws583+FpuldOxbNQPi7FJNH75928/v0WIi/GGh1PJEupgImIiIhItlC9kIU/Q+rTv1lFHGzNbDlxnZZfbeDT3w8Sk5hsdDyRLKECJiIiIiLZhoOdDb2bVmB1f3+aVy1GapqFqRtP0mRMOIt2ndOyRMnxVMBEREREJNvxdHPiu461mN6lNt7uzlyNS6L//D288s1mDl6IMTqeyEMztICtX7+e4OBgPDw8MJlMLFmy5IGfmT17Nr6+vjg5OVGiRAm6du3KtWvX0t9ftGgRtWrVwtXVFWdnZ2rUqMHMmTPv2M+hQ4do3bo1BQsWxNnZmdq1a3PmzJmsPD0REREReUQBlYqyvK8f7z9XiXx2Nuw4/TfPf72BT37dT/RNLUuUnMfQAhYfH4+vry+TJk3K1PhNmzbRsWNHunXrxoEDB1iwYAHbtm2je/fu6WPc3Nz48MMP2bx5M3v37qVLly506dKFFStWpI85fvw4DRs2pHLlyoSFhbF3714+/vhjHB0ds/wcRUREROTRONja8E5AedaE+tPKpwRpFvhp82majAlj/vazpKVpWaLkHIZ+EXNQUBBBQUGZHr9582a8vLzo3bs3AN7e3vTs2ZNRo0aljwkICMjwmT59+vDTTz+xceNGWrRoAcCHH35Iy5YtGT16dPq4cuXKPcKZiIiIiMjj5uGaj0lvPM0bda7yydIDHLscx/u/7GXOtjN8+kJ1nipV0OiIIg9kaAGzVr169Rg8eDDLli0jKCiIy5cvs3DhQlq2bHnX8RaLhbVr1xIZGZle0tLS0vjjjz94//33adGiBbt378bb25tBgwbRpk2bex47KSmJpKSk9NcxMf+sPU5OTiY52djL37ePb3QOyTk0Z8RamjNiLc0ZsZY1c6ZOmYL8+vazzNx6hq/XHifi7A1aT9pIu1ql6B9YnkJO9o87rmQD2envGWsymCzZ5FEyJpOJxYsX37cEASxYsICuXbuSmJhISkoKwcHB/PLLL9jZ2aWPiY6OpmTJkiQlJWFjY8PkyZPp2rUrABcvXqREiRI4OTkxYsQIGjduzPLlyxk8eDDr1q3D39//rscdOnQow4YNu2P7nDlzcHJyevgTFxEREZGHFn0Lfj1tZufVf+6scbK18HzpNOoVtWA2GRxO8oyEhATeeOMNoqOjcXFxue/YHFXADh48SGBgIP369aNFixZERUUxcOBAateuzdSpU9PHpaWlceLECeLi4lizZg2ffvopS5YsISAggAsXLlCyZElef/115syZk/6Z1q1b4+zszM8//3zXY9/tCpinpydXr1594A/5cUtOTmbVqlU0a9YsQxEVuRfNGbGW5oxYS3NGrPWoc2bbqesM//0wkZfiAKju4cKQ5ytT09M1i5NKdpGd/p6JiYnB3d09UwUsRy1BHDlyJA0aNGDgwIEA+Pj44OzsjJ+fHyNGjKBEiRIAmM1mypcvD0CNGjU4dOgQI0eOJCAgAHd3d2xtbalatWqGfVepUoWNGzfe89gODg44ODjcsd3Ozs7wX/ht2SmL5AyaM2ItzRmxluaMWOth50yDCsX4o3cRZm45zbiVR9h/IYZXv9vGq7VK8Z/nKlM4/53/jpPcITv8PWPN8XPU94AlJCRgNmeMbGNjA3DfL+VLS0tLv3plb29P7dq1iYyMzDDmyJEjlClTJosTi4iIiMiTYmtjpksDb9YOCOClp0sBMH/HORqPCeOnv06RkppmcEIRg6+AxcXFcezYsfTXJ0+eJCIiAjc3N0qXLs2gQYM4f/48M2bMACA4OJju3bszZcqU9CWIffv2pU6dOnh4eAD/XCWrVasW5cqVIykpiWXLljFz5kymTJmSfpyBAwfSrl07GjVqlH4P2G+//UZYWNgTPX8RERERyXpFCjgw9lVf3qjryZBfD3DgQgyfLD3A3O1n+fSFatTycjM6ouRhhhawHTt20Lhx4/TX/fv3B6BTp05Mnz6dqKioDF+O3LlzZ2JjY5k4cSKhoaG4urrSpEmTDI+hj4+P55133uHcuXPky5ePypUrM2vWLNq1a5c+pm3btnzzzTeMHDmS3r17U6lSJX755RcaNmz4BM5aRERERJ6EZ8q4sfS9hszZepovVkRyKCqGl7/ZzIs1S/JBy8oULaDvgJUnz9ACFhAQcN+lg9OnT79jW0hICCEhIff8zIgRIxgxYsQDj921a9f0JyOKiIiISO5kYzbRoZ4XLZ8qwZiVkczdfpZFu8+z8uAl+gZWoFN9L+xsctRdOZLDabaJiIiISK5XOL8DI1/0YfE7DfAtVZC4pBRG/HGIVl9tYPPxa0bHkzxEBUxERERE8owanq4sfqcBn7/4FIWc7DhyKY7Xv99CyM+7uRidaHQ8yQNUwEREREQkTzGbTbxWpzTrBgTw5rOlMZngtz0XaDI2jG/Cj3MrRU9LlMdHBUxERERE8iRXJ3tGtHmK395ryNOlXUm4lcrnfx7muS/Xs+HoFaPjSS6lAiYiIiIieVr1kgVZ2Ks+X7zsg3t+e05ciafD1G28PWsn52/cNDqe5DIqYCIiIiKS55nNJl6p5cma0AA61/fCbII/91+k6dgwJq49SlJKqtERJZdQARMRERER+f8K5rNjaOtq/NHbjzpebiQmpzFm5RFajF/PusjLRseTXEAFTERERETkX6qUcGFez2eZ0K4GRQo4cOpaAl2mbeetn3Zw9nqC0fEkB1MBExERERG5C5PJRJuaJVkb6k93P29szSZWH7pE4Lhwxq86QmKyliWK9VTARERERETuo4CjHR+2qsqfffyoX64wSSlpfLnmKM3Gh7Pq4CUsFovRESUHUQETEREREcmECsUKMPutukx8oybFXRw5e/0m3WfsoMv07Zy8Gm90PMkhVMBERERERDLJZDLxvI8Ha0L9eTugHHY2JsIir9Bi/Hq+WHGYhFspRkeUbE4FTERERETESs4Otvznucos79sIvwru3EpNY9K64wSODefPfVFalij3pAImIiIiIvKQyhXJz4yudfjmzWco6ZqPC9GJvD17Fx2mbuPY5Tij40k2pAImIiIiIvIITCYTz1Uvzur+/vRuUh57WzMbj13luQnrGbnsEHFJWpYo/0cFTEREREQkC+Szt6F/80qs6teIppWLkpJm4dv1J2g6Noyley5oWaIAKmAiIiIiIlmqTGFnpnauzdROtSjt5sSlmCR6/7yb17/fQuTFWKPjicFUwEREREREHoOmVYqxsl8j+jeriIOtmS0nrtPyqw18+vtBYhKTjY4nBlEBExERERF5TBztbOjdtAKr+/vToloxUtMsTN14kiZjwlm065yWJeZBKmAiIiIiIo+Zp5sT33aoxU9d6+Dt7szVuCT6z9/DK99s5sCFaKPjyROkAiYiIiIi8oT4VyzC8r5+vP9cJfLZ2bDj9N8Ef72RIb/uJzpByxLzAhUwEREREZEnyMHWhncCyrMm1J9WPiVIs8CMzadpMjaM+dvPkpamZYm5mQqYiIiIiIgBPFzzMemNp5n9Vl3KF83PtfhbvP/LXl6c8hd7z90wOp48JipgIiIiIiIGalDenT/7+PFhyyo429sQcfYGL0zaxODF+/g7/pbR8SSLqYCJiIiIiBjMzsZM90ZlWTsggDY1PLBYYM7WMzQeG8bsradJ1bLEXEMFTEREREQkmyjm4siE12oyr8ezVC5egBsJyXy4eD9tJm1i15m/jY4nWUAFTEREREQkm6lbtjC/hzTkk+CqFHCwZd/5aF6c/BfvL9zD1bgko+PJI1ABExERERHJhmxtzHRp4M3aAQG8/EwpAObvOEeTMWH89NcpUlLTDE4oD0MFTEREREQkGytSwIExr/jyy9v1qObhQkxiCp8sPUDwxE1sP3Xd6HhiJRUwEREREZEc4Jkybix9ryGftqlOwXx2HIqK4ZVvNtNvXgSXYxKNjieZpAImIiIiIpJD2JhNdHi2DOsGBPB6HU9MJli8+zxNxobzw4YTJGtZYranAiYiIiIiksO4Odsz8kUfFr/TAN9SBYlLSmHEH4do9dUGNh+/ZnQ8uQ8VMBERERGRHKqGpyuL32nA5y8+RSEnO45ciuP177cQ8vNuLkZrWWJ2pAImIiIiIpKDmc0mXqtTmnUDAujwbBnMJvhtzwWajA3jm/Dj3ErRssTsRAVMRERERCQXcHWy59M21Vn6XkOeLu1Kwq1UPv/zMM99uZ4NR68YHU/+PxUwEREREZFcpHrJgizsVZ8xr/jint+eE1fi6TB1G2/P2sn5GzeNjpfnqYCJiIiIiOQyZrOJl58pxZrQALo08MLGbOLP/RdpOjaMiWuPkpSSanTEPEsFTEREREQklyqYz45Pgqvxe0hD6ni5kZicxpiVR2gxfj3rDl82Ol6epAImIiIiIpLLVSnhwryez/LlazUoWsCBU9cS6DJ9O2/9tIMz1xKMjpenqICJiIiIiOQBJpOJF2qUZE2oP939vLE1m1h96BKB48MZv+oIiclalvgkqICJiIiIiOQhBRzt+LBVVf7s40f9coW5lZLGl2uOEjgunJUHLmKxWIyOmKupgImIiIiI5EEVihVg9lt1mfTG05Qo6Mi5v2/SY+ZOukzfzsmr8UbHy7VUwERERERE8iiTyUQrnxKs7u/P2wHlsLMxERZ5hRbj1/PFisMk3EoxOmKuowImIiIiIpLHOTvY8p/nKrOibyMaVSzCrdQ0Jq07TuDYcJbti9KyxCykAiYiIiIiIgCULZKfn7rU5tsOz1DSNR8XohN5Z/YuOkzdxrHLsUbHyxVUwEREREREJJ3JZKJFteKs7u9P76YVsLc1s/HYVZ6bsIGRyw4Rl6RliY9CBUxERERERO6Qz96G/s0qsqpfI5pWLkpKmoVv15+g6dgwfo04r2WJD0kFTERERERE7qlMYWemdq7N1E61KO3mxKWYJPrMjeD177cQeVHLEq2lAiYiIiIiIg/UtEoxVvZrRP9mFXGwNbPlxHVafrWB4b8dJCYx2eh4OYahBWz9+vUEBwfj4eGByWRiyZIlD/zM7Nmz8fX1xcnJiRIlStC1a1euXbuW/v6iRYuoVasWrq6uODs7U6NGDWbOnHnP/fXq1QuTycSECROy4IxERERERHIvRzsbejetwOr+/rSoVozUNAs/bjpJkzHh/LLznJYlZoKhBSw+Ph5fX18mTZqUqfGbNm2iY8eOdOvWjQMHDrBgwQK2bdtG9+7d08e4ubnx4YcfsnnzZvbu3UuXLl3o0qULK1asuGN/ixcvZsuWLXh4eGTZOYmIiIiI5Haebk5826EWP3Wtg7e7M1fjkghdsIdXvtnMgQvRRsfL1myNPHhQUBBBQUGZHr9582a8vLzo3bs3AN7e3vTs2ZNRo0aljwkICMjwmT59+vDTTz+xceNGWrRokb79/PnzhISEsGLFClq1avVoJyIiIiIikgf5VyzC8r5+TN14kq/XHGPH6b8J/nojbz5bhtBmlSjoZGd0xGzH0AJmrXr16jF48GCWLVtGUFAQly9fZuHChbRs2fKu4y0WC2vXriUyMjJDSUtLS6NDhw4MHDiQatWqZerYSUlJJCUlpb+OiYkBIDk5meRkY9e83j6+0Tkk59CcEWtpzoi1NGfEWpozOZcZ6N6gDM9XL8bnyyNZtv8SMzaf5rc9FxjYvAIv1SyJ2WzK8uNmpzljTQaTJZss1DSZTCxevJg2bdrcd9yCBQvo2rUriYmJpKSkEBwczC+//IKd3f+16+joaEqWLElSUhI2NjZMnjyZrl27pr8/cuRI1q1bx4oVKzCZTHh5edG3b1/69u17z+MOHTqUYcOG3bF9zpw5ODk5WX2+IiIiIiK5UWS0iV9Omrl085/SVSa/hZe9Uymd3+Bgj1FCQgJvvPEG0dHRuLi43HdsjipgBw8eJDAwkH79+tGiRQuioqIYOHAgtWvXZurUqenj0tLSOHHiBHFxcaxZs4ZPP/2UJUuWEBAQwM6dO2nVqhW7du1Kv/crMwXsblfAPD09uXr16gN/yI9bcnIyq1atolmzZhmKqMi9aM6ItTRnxFqaM2ItzZncJTk1jRlbzvD12uPE30rFZIJXnylFaLPyFHKyz5pjZKM5ExMTg7u7e6YKWI5agjhy5EgaNGjAwIEDAfDx8cHZ2Rk/Pz9GjBhBiRIlADCbzZQvXx6AGjVqcOjQIUaOHElAQAAbNmzg8uXLlC5dOn2/qamphIaGMmHCBE6dOnXXYzs4OODg4HDHdjs7O8N/4bdlpyySM2jOiLU0Z8RamjNiLc2Z3MHODnoFVODFpz3577JDLIm4wLwd51hx8BIDmlfi9TqlscmiZYnZYc5Yc/wc9T1gCQkJmM0ZI9vY2ADc95GXaWlp6VevOnTowN69e4mIiEj/4+HhwcCBA+/6pEQREREREXk4RV0cmfBaTeb1eJbKxQtwIyGZj5bsp82kTew687fR8Qxh6BWwuLg4jh07lv765MmTRERE4ObmRunSpRk0aBDnz59nxowZAAQHB9O9e3emTJmSvgSxb9++1KlTJ3054ciRI6lVqxblypUjKSmJZcuWMXPmTKZMmQJA4cKFKVy4cIYcdnZ2FC9enEqVKj2hMxcRERERyTvqli3M7yENmbnlNONWHmHf+WhenPwXrzxTiv8EVcY9/50rzXIrQwvYjh07aNy4cfrr/v37A9CpUyemT59OVFQUZ86cSX+/c+fOxMbGMnHiREJDQ3F1daVJkyYZnnAYHx/PO++8w7lz58iXLx+VK1dm1qxZtGvX7smdmIiIiIiIZGBrY6ZLA2+e9/Fg1PLDLNx5jgU7z7HiwEVCm1eifd3S2NrkqAV6D8XQAhYQEHDfpYPTp0+/Y1tISAghISH3/MyIESMYMWKEVTnudd+XiIiIiIhkrSIFHBjzii+v1/FkyK8HOHAhhk+WHuDnbWf4tE11anu5GR3xscr9FVNERERERLKdZ8q4sfS9hnzapjoF89lx+GIsr3yzmX7zIrgck2h0vMdGBUxERERERAxhYzbR4dkyrBsQwOt1PDGZYPHu8zQZG84PG06QnJpmdMQspwImIiIiIiKGcnO2Z+SLPix5pwG+pQoSl5TCiD8O0eqrDWw+fs3oeFlKBUxERERERLIFX09XFr/TgM9ffIpCTnYcuRTH699vIeTn3URF30wfl5pmYevJ6+y8amLryeukpt37uRLZTY76Iub/1969x0ZR93sc/8xS2G5rW6nYdikoEKBSkBLCradqw01uaVKDgUofKff00HIgBkRRKQ3kcDSI8VqjgaqxtKHEYh+oBQQFrfAgPBRBK17AE7DcDfTC5aCd84eH5uwDFLaxM7Pt+5Vsws7OwmeSb8p8+pvZBQAAANC6uVyG0obcp7H9YvTy1h9U8I//1t8PVmt71Wn9x8heir3bo/8sq9LJi1cktdMHP+6TNyJYOSnxGtvPa3f822IFDAAAAIDj3B3SQctT+6k0+yENvO9uXfqfP/Rfn3z/f6thvh/SceriFf37h/9U+eGTNqW9cxQwAAAAAI7VLzZCGzL/TS893l8u4+b7XL8AMffv3zn+ckQKGAAAAABHc7kMde0Yoqa6lSnp5MUr2nvsN8tyNQcFDAAAAIDjnam9s+8Gu9P97EIBAwAAAOB4UWHBf+l+dqGAAQAAAHC8Id0j5Y0I1i1uA5MhyRsRrCHdI62M5TcKGAAAAADHa+cylJMSL0k3lLDrz3NS4tXuVp/U4RAUMAAAAAABYWw/r/L+NlAxEb6XGcZEBCvvbwMD4nvA+CJmAAAAAAFjbD+vRsfHaPdPZ7T1i3/o0YeHKrFnlONXvq6jgAEAAAAIKO1choZ2j9T5KlNDu0cGTPmSuAQRAAAAACxDAQMAAAAAi1DAAAAAAMAiFDAAAAAAsAgFDAAAAAAsQgEDAAAAAItQwAAAAADAIhQwAAAAALAIBQwAAAAALEIBAwAAAACLUMAAAAAAwCIUMAAAAACwCAUMAAAAACwSZHeAQGWapiSppqbG5iTStWvXdOnSJdXU1Kh9+/Z2x0EAYGbgL2YG/mJm4C9mBv5y0sxc7wTXO0JTKGDNVFtbK0nq2rWrzUkAAAAAOEFtba0iIiKa3Mcw76Sm4QYNDQ2qrq5WWFiYDMOwNUtNTY26du2q48ePKzw83NYsCAzMDPzFzMBfzAz8xczAX06aGdM0VVtbq86dO8vlavouL1bAmsnlcqlLly52x/ARHh5u+/AhsDAz8BczA38xM/AXMwN/OWVmbrfydR0fwgEAAAAAFqGAAQAAAIBFKGCtgNvtVk5Ojtxut91RECCYGfiLmYG/mBn4i5mBvwJ1ZvgQDgAAAACwCCtgAAAAAGARChgAAAAAWIQCBgAAAAAWoYABAAAAgEUoYAFg165dSklJUefOnWUYhjZu3Hjb93z++ecaOHCg3G63evbsqffee6/Fc8I5/J2Zjz76SKNHj9a9996r8PBwJSYmasuWLdaEhSM05+fMdRUVFQoKCtKAAQNaLB+cpTnzcvXqVT333HO6//775Xa71a1bN61du7blw8IRmjMzBQUFSkhIUEhIiLxer2bMmKHz58+3fFg4wsqVKzV48GCFhYUpKipKqampOnLkyG3fV1xcrAceeEDBwcF68MEHVVZWZkFa/1DAAkB9fb0SEhL05ptv3tH+x44d04QJEzR8+HBVVlZqwYIFmjVrFifUbYi/M7Nr1y6NHj1aZWVl2r9/v4YPH66UlBQdOHCghZPCKfydmesuXLigqVOnauTIkS2UDE7UnHmZNGmStm/frjVr1ujIkSMqLCxUXFxcC6aEk/g7MxUVFZo6dapmzpypb7/9VsXFxdq7d69mz57dwknhFDt37lRWVpb27Nmjbdu26dq1a3r00UdVX19/y/d89dVXeuKJJzRz5kwdOHBAqampSk1N1eHDhy1Mfnt8DH2AMQxDJSUlSk1NveU+ixcv1ubNm32GLS0tTRcuXFB5ebkFKeEkdzIzN9O3b19NnjxZS5cubZlgcCx/ZiYtLU29evVSu3bttHHjRlVWVrZ4PjjLncxLeXm50tLSdPToUUVGRloXDo50JzOzatUq5eXl6eeff27c9vrrr+vFF1/UiRMnLEgJpzl79qyioqK0c+dOPfLIIzfdZ/Lkyaqvr9emTZsatw0bNkwDBgzQ22+/bVXU22IFrBXavXu3Ro0a5bNtzJgx2r17t02JEGgaGhpUW1vLiRKalJ+fr6NHjyonJ8fuKHC40tJSDRo0SC+99JJiY2PVu3dvLVy4UJcvX7Y7GhwqMTFRx48fV1lZmUzT1OnTp7VhwwaNHz/e7miwycWLFyWpyXOTQDkHDrI7AP56p06dUnR0tM+26Oho1dTU6PLly/J4PDYlQ6BYtWqV6urqNGnSJLujwKF+/PFHPfPMM/riiy8UFMR/JWja0aNH9eWXXyo4OFglJSU6d+6c5s6dq/Pnzys/P9/ueHCgpKQkFRQUaPLkybpy5Yp+//13paSk+H2ZNFqHhoYGLViwQElJSerXr98t97vVOfCpU6daOqJfWAED4GPdunXKzc3V+vXrFRUVZXccONAff/yhKVOmKDc3V71797Y7DgJAQ0ODDMNQQUGBhgwZovHjx2v16tV6//33WQXDTX333XeaP3++li5dqv3796u8vFy//PKLMjMz7Y4GG2RlZenw4cMqKiqyO8pfgl9btkIxMTE6ffq0z7bTp08rPDyc1S80qaioSLNmzVJxcfENS/jAdbW1tdq3b58OHDig7OxsSX+eYJumqaCgIG3dulUjRoywOSWcxOv1KjY2VhEREY3b+vTpI9M0deLECfXq1cvGdHCilStXKikpSYsWLZIk9e/fX6GhoXr44Ye1YsUKeb1emxPCKtnZ2dq0aZN27dqlLl26NLnvrc6BY2JiWjKi31gBa4USExO1fft2n23btm1TYmKiTYkQCAoLCzV9+nQVFhZqwoQJdseBg4WHh+vQoUOqrKxsfGRmZiouLk6VlZUaOnSo3RHhMElJSaqurlZdXV3jth9++EEul+u2J1Romy5duiSXy/c0tV27dpIkPj+ubTBNU9nZ2SopKdGOHTvUvXv3274nUM6BWQELAHV1dfrpp58anx87dkyVlZWKjIzUfffdp2effVa//vqrPvjgA0lSZmam3njjDT399NOaMWOGduzYofXr12vz5s12HQIs5u/MrFu3ThkZGXr11Vc1dOjQxmulPR6Pz2+s0Xr5MzMul+uGa/CjoqIUHBzc5LX5aD38/RkzZcoULV++XNOnT1dubq7OnTunRYsWacaMGVyZ0Ub4OzMpKSmaPXu28vLyNGbMGJ08eVILFizQkCFD1LlzZ7sOAxbKysrSunXr9PHHHyssLKzx3CQiIqLx58bUqVMVGxurlStXSpLmz5+v5ORkvfzyy5owYYKKioq0b98+vfPOO7Ydx02ZcLzPPvvMlHTDIyMjwzRN08zIyDCTk5NveM+AAQPMDh06mD169DDz8/Mtzw37+DszycnJTe6P1q85P2f+v5ycHDMhIcGSrLBfc+alqqrKHDVqlOnxeMwuXbqYTz31lHnp0iXrw8MWzZmZ1157zYyPjzc9Ho/p9XrN9PR088SJE9aHhy1uNi+SfM5pk5OTbzhXWb9+vdm7d2+zQ4cOZt++fc3NmzdbG/wO8D1gAAAAAGAR7gEDAAAAAItQwAAAAADAIhQwAAAAALAIBQwAAAAALEIBAwAAAACLUMAAAAAAwCIUMAAAAACwCAUMAAAAACxCAQMAwAaGYWjjxo12xwAAWIwCBgBoc6ZNmybDMG54jB071u5oAIBWLsjuAAAA2GHs2LHKz8/32eZ2u21KAwBoK1gBAwC0SW63WzExMT6Pjh07Svrz8sC8vDyNGzdOHo9HPXr00IYNG3zef+jQIY0YMUIej0f33HOP5syZo7q6Op991q5dq759+8rtdsvr9So7O9vn9XPnzumxxx5TSEiIevXqpdLS0pY9aACA7ShgAADcxAsvvKCJEyfq4MGDSk9PV1pamqqqqiRJ9fX1GjNmjDp27Kivv/5axcXF+vTTT30KVl5enrKysjRnzhwdOnRIpaWl6tmzp8+/kZubq0mTJumbb77R+PHjlZ6ert9++83S4wQAWMswTdO0OwQAAFaaNm2aPvzwQwUHB/tsX7JkiZYsWSLDMJSZmam8vLzG14YNG6aBAwfqrbfe0rvvvqvFixfr+PHjCg0NlSSVlZUpJSVF1dXVio6OVmxsrKZPn64VK1bcNINhGHr++ee1fPlySX+WurvuukuffPIJ96IBQCvGPWAAgDZp+PDhPgVLkiIjIxv/nJiY6PNaYmKiKisrJUlVVVVKSEhoLF+SlJSUpIaGBh05ckSGYai6ulojR45sMkP//v0b/xwaGqrw8HCdOXOmuYcEAAgAFDAAQJsUGhp6wyWBfxWPx3NH+7Vv397nuWEYamhoaIlIAACH4B4wAABuYs+ePTc879OnjySpT58+OnjwoOrr6xtfr6iokMvlUlxcnMLCwtStWzdt377d0swAAOdjBQwA0CZdvXpVp06d8tkWFBSkTp06SZKKi4s1aNAgPfTQQyooKNDevXu1Zs0aSVJ6erpycnKUkZGhZcuW6ezZs5o3b56efPJJRUdHS5KWLVumzMxMRUVFady4caqtrVVFRYXmzZtn7YECAByFAgYAaJPKy8vl9Xp9tsXFxen777+X9OcnFBYVFWnu3Lnyer0qLCxUfHy8JCkkJERbtmzR/PnzNXjwYIWEhGjixIlavXp149+VkZGhK1eu6JVXXtHChQvVqVMnPf7449YdIADAkfgURAAA/oVhGCopKVFqaqrdUQAArQz3gAEAAACARShgAAAAAGAR7gEDAOBfcHU+AKClsAIGAAAAABahgAEAAACARShgAAAAAGARChgAAAAAWIQCBgAAAAAWoYABAAAAgEUoYAAAAABgEQoYAAAAAFjkfwHS/NGoaKB29wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ @@ -447,7 +458,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -460,16 +471,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "What is FHE?\n", - "\n", - "The FSH paradigm is the concept that allows the computation of arbitrary values to an external processor. This technique is useful for many different purposes,\n" + "What is FHE?? I am a doctor and have a Doctor? My doctor is a psychiatrist and I have done a research into my daughter who I read the paper I\n" ] } ], @@ -481,16 +490,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "What is FHE?\n", - "\n", - "FHE is a single-layer, multi-direction, and multi. It is designed to be an efficient, single directional, inter-\n" + "What is FHE?, and I think it is a C-, C, F- and G, I believe it's is not a, O'B, B\n" ] } ], @@ -506,7 +513,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -530,7 +537,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, "outputs": [ { @@ -548,7 +555,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ @@ -563,14 +570,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Total number of weights: 39569664\n", + "Total number of weights: 38783232\n", "Total number of LoRA weights: 147456\n" ] } @@ -581,14 +588,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Total weights removed: 68.24 %\n" + "Total weights removed: 68.87 %\n" ] } ],