ask_pdf_with_gpt4allmodelgguf_cpu (4).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iYNpHKgoCHi3"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install streamlit embedchain"
      ],
      "metadata": {
        "id": "lzdxh_7rCLig"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade embedchain"
      ],
      "metadata": {
        "id": "hoTYbFyBCsY3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q embedchain"
      ],
      "metadata": {
        "id": "ci1owcXLC5HG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install mem0ai"
      ],
      "metadata": {
        "id": "rhXP-_4iDW2u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install langchain"
      ],
      "metadata": {
        "id": "IIV5d8qmD7Xu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install mem0ai"
      ],
      "metadata": {
        "id": "JfdYqN56EMAe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from mem0 import Memory\n",
        "\n",
        "m = Memory()"
      ],
      "metadata": {
        "id": "BzIJfcNuES3-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import streamlit as st\n",
        "import tempfile\n",
        "from embedchain import BotAgent"
      ],
      "metadata": {
        "id": "vqefvg36CLlO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from embedchain import BotAgent\n",
        "from embedchain.models import LLaMA  # or use OpenAI, Cohere, Anthropic\n",
        "# Choose your LLM\n",
        "llm = LLaMA()  # or use OpenAI, Cohere, Anthropic\n",
        "# Choose your vector database\n",
        "from embedchain.vector_stores import Weaviate\n",
        "# Create a Weaviate vector store\n",
        "vector_store = Weaviate()"
      ],
      "metadata": {
        "id": "uK4vDCq7CME4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gpt4all"
      ],
      "metadata": {
        "id": "NUZtUWYHHpM2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from gpt4all import GPT4All\n",
        "model = GPT4All(\"Meta-Llama-3-8B-Instruct.Q4_0.gguf\") # downloads / loads a 4.66GB LLM\n",
        "with model.chat_session():\n",
        "    print(model.generate(\"How can I run LLMs efficiently on my laptop?\", max_tokens=1024))"
      ],
      "metadata": {
        "id": "ccLqNcJDHpmW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from nomic import embed\n",
        "embeddings = embed.text([\"String 1\", \"String 2\"], inference_mode=\"local\")['embeddings']\n",
        "print(\"Number of embeddings created:\", len(embeddings))\n",
        "print(\"Number of dimensions per embedding:\", len(embeddings[0]))"
      ],
      "metadata": {
        "id": "mLp0h9fvIBnu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install langchain_upstage"
      ],
      "metadata": {
        "id": "8RFSV7m9JyFv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pymupdf"
      ],
      "metadata": {
        "id": "RGJSHAcHKPCW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "\n",
        "data = loader.load()\n",
        "data[4]"
      ],
      "metadata": {
        "id": "7E44oxXeJywG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install --upgrade --quiet  langchain langchain-community\n"
      ],
      "metadata": {
        "id": "A9JTaAMRKiEW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "pip install faiss-cpu"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "Ohm36U3eLXqm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "# تحميل الكتاب باستخدام PyMuPDFLoader\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج جميع صفحات الكتاب في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة باستخدام RecursiveCharacterTextSplitter\n",
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=1000, chunk_overlap=200\n",
        ")\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# حفظ الأجزاء في مخزن فايس (محاكاة فقط)\n",
        "# في الواقع ، ستحتاج إلى الاتصال بواجهة برمجة تطبيقات فايس لحفظ البيانات.\n",
        "print(\"حفظ الأجزاء في مخزن فايس ...\")\n",
        "for i, chunk in enumerate(chunks):\n",
        "    print(f\"حفظ الجزء {i + 1} ...\")\n",
        "    # هنا يمكنك استخدام واجهة برمجة تطبيقات فايس لحفظ chunk\n",
        "\n",
        "print(\"تم حفظ جميع الأجزاء بنجاح.\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "POj0yBq7LU1I"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "import faiss\n",
        "\n",
        "# تحديد عدد الأبعاد لمتجهات النص\n",
        "d = 128  # يمكن تعديلها حسب احتياجاتك\n",
        "\n",
        "# إنشاء فهرس فايس\n",
        "index = faiss.IndexFlatL2(d)"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "r5uIrBRjLZFu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "\n",
        "# تحميل نموذج OpenAI embeddings لتحويل النص إلى متجهات\n",
        "embeddings = OpenAIEmbeddings()\n",
        "\n",
        "# تحويل أجزاء الكتاب إلى متجهات\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "\n",
        "# إضافة المتجهات إلى فهرس فايس\n",
        "index.add(np.array(chunk_vectors).astype('float32'))\n",
        "\n",
        "# حفظ فهرس فايس\n",
        "faiss.write_index(index, \"book_index.bin\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "fJu3NRGOLZcu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "!pip install langchain_community\n"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "ps40AlU3Lsym"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "from langchain_community.embeddings import HuggingFaceEmbeddings"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "gy0txoNGLtYW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "biCwgvYlLtuG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "!pip install langchain_community\n",
        "\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "\n",
        "# تحديد نموذج هاججفيس\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# تحويل أجزاء الكتاب إلى متجهات\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "\n",
        "# إضافة المتجهات إلى فهرس فايس\n",
        "index.add(np.array(chunk_vectors).astype('float32'))\n",
        "\n",
        "\n",
        "# حفظ فهرس فايس\n",
        "faiss.write_index(index, \"book_index.bin\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "7VK_EkxZLuUV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "!pip install sentence-transformers"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "4Sxif2bSLvjO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "# تحميل فهرس فايس\n",
        "index = faiss.read_index(\"book_index.bin\")\n",
        "\n",
        "# تحويل النص المراد البحث عنه إلى متجه\n",
        "query_vector = embeddings.embed_query(\"What is the title of the book?\")\n",
        "\n",
        "# البحث عن المتجهات المشابهة في فهرس فايس\n",
        "D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "# عرض النتائج\n",
        "for i in I[0]:\n",
        "    print(chunks[i])"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "YJUhCQXVLafW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Nt9B7tsyMPDu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "import faiss\n",
        "import os\n",
        "\n",
        "#Check if the index file exists\n",
        "index_file_path = \"book_index.bin\"\n",
        "if os.path.exists(index_file_path):\n",
        "  # Load the Faiss index if it exists\n",
        "  index = faiss.read_index(index_file_path)\n",
        "  print(\"Index loaded successfully.\")\n",
        "else:\n",
        "  print(f\"Error: Index file '{index_file_path}' not found. Please create the index first.\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "FDj2fMmTMmFn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "!pip install faiss-cpu\n",
        "!pip install langchain_community\n",
        "!pip install sentence-transformers"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "SoKQRBEJMtv-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "import faiss\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "6Zt6FqGnMt7Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "# Load the PDF document\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# Combine pages into a single text\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# Split text into chunks\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "H3BFtQieMu8p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "import numpy as np\n",
        "# Initialize embeddings\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# Create the FAISS index\n",
        "d = embeddings.embed_query(chunks[0]).shape[0]  # Get embedding dimension\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# Convert chunks to embeddings and add to index\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "index.add(np.array(chunk_vectors).astype('float32'))\n",
        "\n",
        "# Save the index to a file\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "print(\"Index created and saved to book_index.bin\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "12gUPJL1MvjI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0] # Convert to numpy array first\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')  # تحويل إلى مصفوفة NumPy\n",
        "index.add(chunk_vectors_np)  # إضافة إلى فهرس Faiss\n",
        "\n",
        "# حفظ الفهرس إلى ملف\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "print(\"تم إنشاء الفهرس وحفظه في book_index.bin\")\n",
        "\n",
        "# تحميل الفهرس\n",
        "index = faiss.read_index(\"book_index.bin\")\n",
        "print(\"تم تحميل الفهرس بنجاح\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "M7UPV5TrNFof"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "# Load the index\n",
        "index = faiss.read_index(\"book_index.bin\")\n",
        "print(\"Index loaded successfully\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "HUqdO_CxMwEX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gpt4all\n",
        "!pip install faiss-cpu\n",
        "!pip install langchain_community\n",
        "!pip install sentence-transformers"
      ],
      "metadata": {
        "id": "wADlch8jNxMO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gpt4all"
      ],
      "metadata": {
        "id": "UT8e5xC4NywO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "\n",
        "\n",
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "# ... (كود تحميل وتحضير النص كما هو) ...\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# ... (كود إنشاء فهرس Faiss وحفظه كما هو) ...\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(\"ggml-gpt4all-j-v1.3-groovy.bin\")\n",
        "\n",
        "# قالب لتوليد السؤال\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Generate a question that can be answered using the information in the text.\n",
        "The question should be relevant and interesting.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء سؤال\n",
        "def generate_question(query):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد السؤال باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        question = model.generate(prompt.format(context=context), max_tokens=50)\n",
        "\n",
        "    return question\n",
        "\n",
        "# مثال على استخدام الدالة\n",
        "question = generate_question(\"What is the main character's name?\")\n",
        "\n",
        "print(question)"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "AU6P2uRdNtXo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "BffAEdmtMfiI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "!pip install faiss-cpu\n",
        "!pip install langchain_community\n",
        "!pip install sentence-transformers\n",
        "!pip install gpt4all"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "eEh3Z7pZOqtP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0] # Convert to numpy array first\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')  # تحويل إلى مصفوفة NumPy\n",
        "index.add(chunk_vectors_np)  # إضافة إلى فهرس Faiss\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "# model = GPT4All(\"ggml-gpt4all-j-v1.3-groovy.bin\")  # إذا كان النموذج محليًا\n",
        "\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "model = GPT4All()  # لتحميل النموذج الافتراضي\n",
        "\n",
        "# قالب لتوليد السؤال\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Generate a question that can be answered using the information in the text.\n",
        "The question should be relevant and interesting.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء سؤال\n",
        "def generate_question(query):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد السؤال باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        question = model.generate(prompt.format(context=context), max_tokens=50)\n",
        "\n",
        "    return question\n",
        "\n",
        "# مثال على استخدام الدالة\n",
        "question = generate_question(\"What is the main character's name?\")\n",
        "\n",
        "print(question)"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "DG5jPDFKOq72"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "model = GPT4All(model_name=\"ggml-gpt4all-j-v1.3-groovy.bin\")"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "8GSxMlG7O7v_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://huggingface.co/TheBloke/Mistral-7B-v0.1-GGUF/resolve/main/mistral-7b-v0.1.Q2_K.gguf"
      ],
      "metadata": {
        "id": "kPVwIx0yMflf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "PsAehEQnMfpg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "wpI-KdEqQSby"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ob-frbhMQSfe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cMz4blp8QSh-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "XntOuqgVQSkr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "شغال"
      ],
      "metadata": {
        "id": "cxxGp6eGSGHo"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cMh25Ms7Mfqz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "39lS9iGVMftx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "hyeSU6BHMfwf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "pTK4zvoYMfyi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "# تحميل الكتاب باستخدام PyMuPDFLoader\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج جميع صفحات الكتاب في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة باستخدام RecursiveCharacterTextSplitter\n",
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=1000, chunk_overlap=200\n",
        ")\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# حفظ الأجزاء في مخزن فايس (محاكاة فقط)\n",
        "# في الواقع ، ستحتاج إلى الاتصال بواجهة برمجة تطبيقات فايس لحفظ البيانات.\n",
        "print(\"حفظ الأجزاء في مخزن فايس ...\")\n",
        "for i, chunk in enumerate(chunks):\n",
        "    print(f\"حفظ الجزء {i + 1} ...\")\n",
        "    # هنا يمكنك استخدام واجهة برمجة تطبيقات فايس لحفظ chunk\n",
        "\n",
        "print(\"تم حفظ جميع الأجزاء بنجاح.\")\n",
        "import faiss\n",
        "\n",
        "# تحديد عدد الأبعاد لمتجهات النص\n",
        "d = 128  # يمكن تعديلها حسب احتياجاتك\n",
        "\n",
        "# إنشاء فهرس فايس\n",
        "index = faiss.IndexFlatL2(d)\n",
        "!pip install langchain_community\n",
        "\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "\n",
        "# تحديد نموذج هاججفيس\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# تحويل أجزاء الكتاب إلى متجهات\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "\n",
        "# إضافة المتجهات إلى فهرس فايس\n",
        "index.add(np.array(chunk_vectors).astype('float32'))\n",
        "\n",
        "\n",
        "# حفظ فهرس فايس\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "# تحميل فهرس فايس\n",
        "index = faiss.read_index(\"book_index.bin\")\n",
        "\n",
        "# تحويل النص المراد البحث عنه إلى متجه\n",
        "query_vector = embeddings.embed_query(\"What is the title of the book?\")\n",
        "\n",
        "# البحث عن المتجهات المشابهة في فهرس فايس\n",
        "D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "# عرض النتائج\n",
        "for i in I[0]:\n",
        "    print(chunks[i])"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "5KEBGOUhMgFB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "شغال ع المعالج"
      ],
      "metadata": {
        "id": "AtKSP73qWVe3"
      }
    },
    {
      "source": [
        "\n",
        "\n",
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0] # Convert to numpy array first\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')  # تحويل إلى مصفوفة NumPy\n",
        "index.add(chunk_vectors_np)  # إضافة إلى فهرس Faiss\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد السؤال\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Generate a question that can be answered using the information in the text.\n",
        "The question should be relevant and interesting.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء سؤال\n",
        "def generate_question(query):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد السؤال باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        question = model.generate(prompt.format(context=context), max_tokens=50)\n",
        "\n",
        "    return question\n",
        "\n",
        "# مثال على استخدام الدالة\n",
        "question = generate_question(\"What is the main character's name?\")\n",
        "\n",
        "print(question)"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "yEFKgJc6QT74"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "rDEMhAmkWa93"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "AqBFsbdoWa6v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0] # Convert to numpy array first\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')  # تحويل إلى مصفوفة NumPy\n",
        "index.add(chunk_vectors_np)  # إضافة إلى فهرس Faiss\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد السؤال\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Generate a question that can be answered using the information in the text.\n",
        "The question should be relevant and interesting.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء سؤال\n",
        "def generate_question(query):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد السؤال باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        question = model.generate(prompt.format(context=context), max_tokens=512)\n",
        "\n",
        "    return question\n",
        "\n",
        "# مثال على استخدام الدالة\n",
        "question = generate_question(\"What is the topic of the book?\")\n",
        "\n",
        "print(question)"
      ],
      "metadata": {
        "id": "QuGXuTU1Wa3f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0] # Convert to numpy array first\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')  # تحويل إلى مصفوفة NumPy\n",
        "index.add(chunk_vectors_np)  # إضافة إلى فهرس Faiss\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد السؤال\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Generate a question that can be answered using the information in the text.\n",
        "The question should be relevant and interesting.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء سؤال\n",
        "def generate_question(query):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد السؤال باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        question = model.generate(prompt.format(context=context), max_tokens=512)\n",
        "\n",
        "    return question\n",
        "\n",
        "# مثال على استخدام الدالة\n",
        "question = generate_question(\"What is the main problem in the story?\")\n",
        "\n",
        "print(question)"
      ],
      "metadata": {
        "id": "nGioGDVKcbvb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "6DMlb5ZQczhu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "NUHMeK_TkXUg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "dmpD98mSkXR3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0] # Convert to numpy array first\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')  # تحويل إلى مصفوفة NumPy\n",
        "index.add(chunk_vectors_np)  # إضافة إلى فهرس Faiss\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد السؤال\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Create an answer using the information in the text.\n",
        "The answer should be relevant to the question.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء سؤال\n",
        "def generate_question(query):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد السؤال باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        question = model.generate(prompt.format(context=context), max_tokens=512)\n",
        "\n",
        "    return question\n",
        "\n",
        "# مثال على استخدام الدالة\n",
        "question = generate_question(\"who is Grover?\")\n",
        "\n",
        "print(question)"
      ],
      "metadata": {
        "id": "Ues4_nhCkXOg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "شغال اجابة علىى سؤالى"
      ],
      "metadata": {
        "id": "w3hgSCr20L0u"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Answer the following question using the information in the text:\n",
        "\n",
        "{question}\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=5)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=10)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "JiVisAbwshUC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf /content/book_index.bin"
      ],
      "metadata": {
        "id": "-awVEWbq0bRZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Based on the text, please provide a detailed answer to the following question:\n",
        "\n",
        "{question}\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=5, max_tokens_value=700):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question, k_value=7, max_tokens_value=700) #زيادة قيمة k و max_tokens\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "jEnyoKW52ofS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "What is Grover’s relationship to Percy Jackson’s?"
      ],
      "metadata": {
        "id": "ZPGvJijI3UcR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "رائع شغال\n",
        "السؤال: من هو جروفر؟\n",
        "الإجابة:\n",
        "يحتوي النص على إشارة إلى فرد يُدعى جروفر. يظهر الاسم في مكانين في النص ويذكره شخص آخر. وهذا يشير إلى أن جروفر هو شخص معروف لكلا الشخصين، ولكن ليس بالضرورة للقارئ. يمكننا أن نستنتج من هذا السياق أن جروفر هو على الأرجح صديق أو معارف لأحد الفردين أو كليهما.\n",
        "\n",
        "بناءً على النص، أجب عن السؤال التالي:\n",
        "\n",
        "من هو جروفر؟\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "QZ56Zgfl8eNz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Based on the text, answer the following question:\n",
        "\n",
        "{question}\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=3, max_tokens_value=300): #تم التعديل هنا\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "oNBqNmgp6pqr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "تحسين"
      ],
      "metadata": {
        "id": "46JcGLP-83v6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Using only the provided text, directly answer the question: {question}\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=3, max_tokens_value=300): #تم التعديل هنا\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "t59sTC8285T6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Based on the text, summarize Grover's personality, physical appearance, and any relevant information about him.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=3, max_tokens_value=300):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "vh0zAzct_sjE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Based on the text, answer the question briefly.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "\n",
        "definition_template = \"\"\"\n",
        "Explain the meaning of the following word: {word}\n",
        "\n",
        "definition: {definition}\n",
        "\"\"\"\n",
        "\n",
        "definition_prompt = PromptTemplate(\n",
        "    input_variables=[\"word\", \"definition\"],\n",
        "    template=definition_template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=3, max_tokens_value=300):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "\n",
        "        if query == \"what is assistant\":\n",
        "           answer = model.generate(definition_prompt.format(word=\"assistant\", definition=assistant_definition),max_tokens=max_tokens_value)\n",
        "        else:\n",
        "             answer = model.generate(prompt.format(context=context), max_tokens=max_tokens_value)\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")\n",
        "\n",
        "question2 = \"what is assistant\"\n",
        "answer2 = generate_answer(question2)\n",
        "print(f\"Question: {question2}\")\n",
        "print(f\"Answer: {answer2}\")"
      ],
      "metadata": {
        "id": "CBayxviMHT2k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "hd3gQkGpJNVK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "0dcjT_oCJNS5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "oYujsHFjJNOi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "_SnL0gBTJNMC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "CFjudpjTJNJP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "gafo3UgsJNF0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "prompt_template = \"\"\"\n",
        "<|start_header_id|>user<|end_header_id|>\n",
        "You are an assistant for answering questions using provided context.\n",
        "You are given the extracted parts of a long document and a question. Provide a conversational answer.\n",
        "If you don't know the answer, just say \"I do not know.\" Don't make up an answer.\n",
        "Question: {question}\n",
        "Context: {context}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=prompt_template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=3, max_tokens_value=300): #تم التعديل هنا\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "prompt_template = \"\"\"\n",
        "<|start_header_id|>user<|end_header_id|>\n",
        "You are an assistant for answering questions using provided context.\n",
        "You are given the extracted parts of a long document and a question. Provide a conversational answer.\n",
        "If you don't know the answer, just say \"I do not know.\" Don't make up an answer.\n",
        "Question: {question}\n",
        "Context: {context}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=prompt_template,\n",
        ")\n",
        "\n",
        "\n",
        "qa_chain = RetrievalQA.from_chain_type(\n",
        "    llm,\n",
        "    retriever=retriever,\n",
        "    chain_type_kwargs={\"prompt\": prompt}\n",
        ")\n",
        "\n",
        "\n",
        "question = \"Are aphids a pest?\"\n",
        "\n",
        "qa_chain.invoke(question)['result']"
      ],
      "metadata": {
        "id": "7JNb0kdSJM-s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "شغال بس وحش"
      ],
      "metadata": {
        "id": "0zgOQg8SMuWw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "prompt_template = \"\"\"\n",
        "<|start_header_id|>user<|end_header_id|>\n",
        "You are an assistant for answering questions using provided context.\n",
        "You are given the extracted parts of a long document and a question. Provide a conversational answer.\n",
        "If you don't know the answer, just say \"I do not know.\" Don't make up an answer.\n",
        "Question: {question}\n",
        "Context: {context}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=prompt_template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=3, max_tokens_value=300): #تم التعديل هنا\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "lWDr-WZ7JxAz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF/resolve/main/Mistral-7B-Instruct-v0.3-Q8_0.gguf"
      ],
      "metadata": {
        "id": "W06png80Kkbj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hkunlp/instructor-large\n",
        "\n",
        "\n",
        "What is the content of the book?"
      ],
      "metadata": {
        "id": "qrovXf_pL4iM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Is Grover a character in the book?"
      ],
      "metadata": {
        "id": "uHOsWgaHOrkB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "prompt_template = \"\"\"\n",
        "<|start_header_id|>user<|end_header_id|>\n",
        "You are an assistant for answering questions using provided context.\n",
        "You are given the extracted parts of a long document and a question. Provide a conversational answer.\n",
        "If you don't know the answer, just say \"I do not know.\" Don't make up an answer.\n",
        "Question: {question}\n",
        "Context: {context}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=prompt_template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=3, max_tokens_value=300): #تم التعديل هنا\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"What's in the book?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "qwSb-8RIMRJK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "شغال افضل"
      ],
      "metadata": {
        "id": "6wy_JPTwPXFM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=100)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/mistral-7b-v0.1.Q2_K.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "template = \"\"\"\n",
        "Given the following text from a book:\n",
        "\n",
        "{context}\n",
        "\n",
        "Using only the provided text, directly answer the question: {question}.\n",
        "Do not use any outside information or prior knowledge.\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=5, max_tokens_value=300):\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "\n",
        "      prompt_with_instruction = prompt.format(context=context, question=query)\n",
        "      answer = model.generate(prompt_with_instruction, max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"What's in the book?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "YoVReK_6KJ_l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qa_chain = RetrievalQA.from_chain_type(\n",
        "    llm,\n",
        "    retriever=retriever,\n",
        "    chain_type_kwargs={\"prompt\": prompt}\n",
        ")"
      ],
      "metadata": {
        "id": "icKWdPMXOAUi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "question = \"Are aphids a pest?\"\n",
        "\n",
        "qa_chain.invoke(question)['result']"
      ],
      "metadata": {
        "id": "yJuFWel_ODmi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "اجابة رائعة وشغال ممتاز"
      ],
      "metadata": {
        "id": "XyCuSGizS-bE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=700, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"hkunlp/instructor-large\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/Mistral-7B-Instruct-v0.3-Q8_0.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "prompt_template = \"\"\"\n",
        "<|start_header_id|>user<|end_header_id|>\n",
        "You are an assistant for extracting information from a given context. Your task is to answer questions based solely on the context provided.\n",
        "If the answer cannot be found in the context, respond with \"I do not know.\". Do not provide answers from outside knowledge.\n",
        "Question: {question}\n",
        "Context: {context}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=prompt_template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=5, max_tokens_value=300): #تم التعديل هنا\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"who is Grover?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "9YskCai5M_6S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "شغال رائع"
      ],
      "metadata": {
        "id": "92D7Q3ZfUvn9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/q.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=700, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"hkunlp/instructor-large\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/Mistral-7B-Instruct-v0.3-Q8_0.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "prompt_template = \"\"\"\n",
        "<|start_header_id|>user<|end_header_id|>\n",
        "You are an assistant for extracting information from a given context. Your task is to answer questions based solely on the context provided.\n",
        "If the answer cannot be found in the context, respond with \"I do not know.\". Do not provide answers from outside knowledge.\n",
        "Question: {question}\n",
        "Context: {context}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=prompt_template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=5, max_tokens_value=300): #تم التعديل هنا\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"Where was the trip?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "TtEVL1IKTWP0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "What is the advice mentioned at the beginning of the book?"
      ],
      "metadata": {
        "id": "TfKnskX8UG_B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain import PromptTemplate, LLMChain\n",
        "from gpt4all import GPT4All\n",
        "\n",
        "# تحميل وثيقة PDF\n",
        "loader = PyMuPDFLoader(\"/content/The_Lightning_Thief_-_Percy_Jackson_1-10.pdf\")\n",
        "data = loader.load()\n",
        "\n",
        "# دمج الصفحات في نص واحد\n",
        "text = \" \".join([page.page_content for page in data])\n",
        "\n",
        "# تقسيم النص إلى أجزاء متداخلة (تم التعديل)\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=700, chunk_overlap=150)\n",
        "chunks = text_splitter.split_text(text)\n",
        "\n",
        "# تهيئة نموذج التضمين\n",
        "embeddings = HuggingFaceEmbeddings(model_name=\"hkunlp/instructor-large\")\n",
        "\n",
        "# إنشاء فهرس Faiss\n",
        "first_chunk_embedding = embeddings.embed_query(chunks[0])\n",
        "d = np.array(first_chunk_embedding).shape[0]\n",
        "\n",
        "index = faiss.IndexFlatL2(d)\n",
        "\n",
        "# تحويل الأجزاء إلى تضمينات وإضافتها إلى الفهرس\n",
        "chunk_vectors = [embeddings.embed_query(chunk) for chunk in chunks]\n",
        "chunk_vectors_np = np.array(chunk_vectors).astype('float32')\n",
        "index.add(chunk_vectors_np)\n",
        "\n",
        "# حفظ الفهرس إلى ملف (اختياري)\n",
        "faiss.write_index(index, \"book_index.bin\")\n",
        "\n",
        "# تحميل نموذج GPT4All\n",
        "model = GPT4All(model_name=\"/content/Mistral-7B-Instruct-v0.3-Q8_0.gguf\")\n",
        "\n",
        "# قالب لتوليد الإجابة (تم التعديل)\n",
        "prompt_template = \"\"\"\n",
        "<|start_header_id|>user<|end_header_id|>\n",
        "You are an assistant for extracting information from a given context. Your task is to answer questions based solely on the context provided.\n",
        "If the answer cannot be found in the context, respond with \"I do not know.\". Do not provide answers from outside knowledge.\n",
        "Question: {question}\n",
        "Context: {context}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
        "\"\"\"\n",
        "\n",
        "prompt = PromptTemplate(\n",
        "    input_variables=[\"context\", \"question\"],\n",
        "    template=prompt_template,\n",
        ")\n",
        "\n",
        "# دالة لإنشاء الإجابة (تم التعديل)\n",
        "def generate_answer(query, k_value=5, max_tokens_value=300): #تم التعديل هنا\n",
        "    # تحويل النص المراد البحث عنه إلى متجه\n",
        "    query_vector = embeddings.embed_query(query)\n",
        "\n",
        "    # البحث عن المتجهات المشابهة في فهرس فايس\n",
        "    D, I = index.search(np.array([query_vector]).astype('float32'), k=k_value)\n",
        "\n",
        "    # استخراج السياق من الأجزاء المطابقة\n",
        "    context = \" \".join([chunks[i] for i in I[0]])\n",
        "\n",
        "    # توليد الإجابة باستخدام GPT4All\n",
        "    with model.chat_session():\n",
        "        answer = model.generate(prompt.format(context=context, question=query), max_tokens=max_tokens_value)\n",
        "\n",
        "    return answer\n",
        "\n",
        "# مثال على استخدام الدالة (تم التعديل)\n",
        "question = \"What is the advice mentioned at the beginning of the book?\"\n",
        "answer = generate_answer(question)\n",
        "\n",
        "print(f\"Question: {question}\")\n",
        "print(f\"Answer: {answer}\")"
      ],
      "metadata": {
        "id": "_5vmHjV3UkxK"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}