AllyArc_Dataset_Chat.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HV1DyBd2RHrN"
      },
      "source": [
        "### Install required pacakges\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "id": "xOj-BuKARJrH"
      },
      "outputs": [],
      "source": [
        "!pip install --q crewai\n",
        "!pip install --q -U duckduckgo-search\n",
        "!pip install --q langchain_google_genai"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "id": "Qa5lBBYFd7Zk"
      },
      "outputs": [],
      "source": [
        "!pip install --upgrade --quiet  html2text\n",
        "\n",
        "import os\n",
        "import json\n",
        "import requests\n",
        "\n",
        "from langchain_community.document_loaders import AsyncHtmlLoader\n",
        "from langchain_community.document_transformers import Html2TextTransformer\n",
        "from langchain.tools import tool\n",
        "from langchain_google_genai import ChatGoogleGenerativeAI\n",
        "from crewai import Agent, Task, Crew, Process\n",
        "from crewai import Agent\n",
        "\n",
        "from IPython.display import Markdown\n",
        "import json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "id": "_mSLIE-TG8VN"
      },
      "outputs": [],
      "source": [
        "@tool\n",
        "def html_to_text(url: str) -> str:\n",
        "    \"\"\"Fetches HTML content from a given URL and converts it to plain text.\n",
        "    The input to this tool should be a URL as a string.\n",
        "    The output will be the converted plain text.\"\"\"\n",
        "    loader = AsyncHtmlLoader([url])\n",
        "    docs = loader.load()  # Load the HTML content\n",
        "\n",
        "    html2text = Html2TextTransformer()\n",
        "    docs_transformed = html2text.transform_documents(docs)  # Convert HTML to text\n",
        "\n",
        "    return docs_transformed[0].page_content if docs_transformed else \"Error in transformation\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "id": "-jAOVSSISABN"
      },
      "outputs": [],
      "source": [
        "# Set gemini pro as llm\n",
        "llm = ChatGoogleGenerativeAI(model=\"gemini-pro\",\n",
        "                             verbose = True,\n",
        "                             temperature = 0.6,\n",
        "                             google_api_key=\"AIzaSyDNYOtMdFWosRJZU7LqrqVMv614zLNONio\")\n",
        "# AIzaSyDb16frvMCRvNBNvTYiBLClRzn05_9sGYE"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 94,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SLDSHOuCUXna",
        "outputId": "f175e890-7395-45a9-b76f-3df0849869b8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[DEBUG]: Working Agent: Senior Web Research Analyst\n",
            "\n",
            "[INFO]: Starting Task: \n",
            "        Your task is to meticulously search and scrape web pages for content related to both teaching strategies and communication techniques for autistic students.\n",
            "        Focus on identifying articles, studies, and resources that offer insights into effective educational practices and mentorship guidance.\n",
            "        Use the html_to_text tool to convert the content of the following URL to plain text: https://www.autism.org.uk/advice-and-guidance/topics/communication/tips\n",
            "        Extract key points that offer insights into both educational strategies and effective ways to communicate and mentor autistic students, ensuring the content's relevance and accuracy.\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new CrewAgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? Yes\n",
            "Action: html_to_text\n",
            "Action Input: https://www.autism.org.uk/advice-and-guidance/topics/communication/tips\u001b[0m"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Fetching pages: 100%|##########| 1/1 [00:00<00:00,  1.39it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[36;1m\u001b[1;3mYour Internet Explorer 11 browser is not supported by this site. Please\n",
            "upgrade to a recent browser for the best experience.\n",
            "\n",
            "Skip to main content\n",
            "\n",
            "#\n",
            "\n",
            "  * Community \n",
            "  * Work for us \n",
            "  * News \n",
            "  * Shop \n",
            "  * Accessibility options \n",
            "\n",
            "Donate\n",
            "\n",
            "  * Advice and guidance \n",
            "    * Advice and guidance \n",
            "      * Advice and guidance \n",
            "      * Autism Services Directory \n",
            "      * Community \n",
            "      * Professional practice \n",
            "      * Spectrum Live \n",
            "      * Stories from the Spectrum \n",
            "      * The Spectrum magazine \n",
            "        * The Spectrum magazine \n",
            "        * Previous editions \n",
            "        * Spectrum Collections \n",
            "      * What is autism \n",
            "        * What is autism \n",
            "        * Autism and BAME people \n",
            "        * Autism and gender identity \n",
            "        * Autistic women and girls \n",
            "        * The causes of autism \n",
            "        * The history of autism \n",
            "        * Varying support needs \n",
            "    *   * What we do \n",
            "    * What we do \n",
            "      * What we do \n",
            "      * Adult residential services \n",
            "        * Adult residential services \n",
            "        * Adult residential services \n",
            "        * Adult services stories \n",
            "        * Supported living services \n",
            "        * Adult residential and supported living service vacancies \n",
            "      * Autism training and best practice \n",
            "        * Autism training and best practice \n",
            "        * Training \n",
            "        * Autism Accreditation \n",
            "        * Diagnostic services \n",
            "        * Consultancy services \n",
            "        * Conferences \n",
            "      * Branches \n",
            "      * Campaign \n",
            "        * Campaign \n",
            "        * About our campaign work \n",
            "        * Changing the law \n",
            "        * Education \n",
            "        * Mental Health \n",
            "        * Sign up to Campaigner Updates \n",
            "      * Diagnostic services \n",
            "      * Education and children's services \n",
            "        * Education and children's services \n",
            "        * Cullum Centres \n",
            "        * Our family of schools \n",
            "      * Employment \n",
            "        * Employment \n",
            "        * Autism at Work programme \n",
            "        * Job opportunities \n",
            "      * Help and support \n",
            "      * Support in the community \n",
            "        * Support in the community \n",
            "        * Centres \n",
            "        * Family support \n",
            "        * Outreach \n",
            "        * Social groups \n",
            "    *   * Who we are \n",
            "    * Who we are \n",
            "      * Who we are \n",
            "      * Northern Ireland \n",
            "      * Scotland \n",
            "      * Wales \n",
            "      * Who we are \n",
            "        * Who we are \n",
            "        * Our leadership team \n",
            "        * Our patron, president and vice presidents \n",
            "        * Our trustees \n",
            "        * Our mission, vision and values \n",
            "        * Transparency and compliance \n",
            "      * Work for us \n",
            "        * Work for us \n",
            "        * Autistic employees \n",
            "        * School based roles \n",
            "        * Support worker roles \n",
            "    *   * How you can help \n",
            "    * How you can help \n",
            "      * How you can help \n",
            "      * Autism insight panels \n",
            "      * Campaign \n",
            "        * Campaign \n",
            "        * About our campaign work \n",
            "        * Sign up to Campaigner Updates \n",
            "        * Changing the law \n",
            "        * Transforming Care \n",
            "      * Corporate \n",
            "        * Corporate \n",
            "        * Our Marketplace community \n",
            "        * Our partners \n",
            "      * Donate \n",
            "        * Donate \n",
            "        * Donate while you shop \n",
            "        * Gift Aid and making your donation go further \n",
            "        * Give in memory \n",
            "        * Leave a gift in your will \n",
            "        * Lottery – play now to win £25,000 \n",
            "        * Other ways to donate \n",
            "        * Payroll Giving \n",
            "      * Membership \n",
            "        * Membership \n",
            "        * Membership FAQs \n",
            "      * Philanthropy \n",
            "        * Philanthropy \n",
            "        * How your support can help change lives \n",
            "      * Raise money \n",
            "        * Raise money \n",
            "        * Fundraising events \n",
            "        * World Autism Acceptance Week 2024 \n",
            "        * Do your own thing \n",
            "      * Volunteer \n",
            "      * Work for us \n",
            "        * Work for us \n",
            "        * Autistic employees \n",
            "        * School based roles \n",
            "        * Support worker roles \n",
            "    * \n",
            "\n",
            "Donate\n",
            "\n",
            "Donate\n",
            "\n",
            "  1.   2. Advice and guidance\n",
            "  3. Communication\n",
            "  4. Communication tips\n",
            "\n",
            "##  \n",
            "Advice & Support for:\n",
            "\n",
            "### In this guide\n",
            "\n",
            "Share guide:\n",
            "\n",
            "# Communication tips\n",
            "\n",
            "**Our quick tips  may help you to communicate more effectively with an\n",
            "autistic person, whether it's your child, pupil, colleague or friend. **\n",
            "\n",
            "##  \n",
            "Getting and keeping their attention  \n",
            "\n",
            "  * **Always use their name** at the beginning so that they know you are talking to them.\n",
            "\n",
            "  * **Make sure they are paying attention** before you ask a question or give an instruction. The signs that someone is paying attention will be different for different people.\n",
            "\n",
            "  * **Use their hobbies and interests** , or the activity they are currently doing, to engage them.  \n",
            "\n",
            "##  \n",
            "Processing information\n",
            "\n",
            "An autistic person can find it difficult to filter out the less important\n",
            "information. If there is too much information, it can lead to 'overload',\n",
            "where no further information can be processed. To help:\n",
            "\n",
            "  * say less and say it slowly\n",
            "\n",
            "  * use specific key words, repeating and stressing them\n",
            "\n",
            "  * pause between words and phrases to give the person time to process what you've said, and to give them a chance to think of a response\n",
            "\n",
            "  * don't use too many questions\n",
            "\n",
            "  * use less non-verbal communication (eg eye contact, facial expressions, gestures, body language)\n",
            "\n",
            "  * use visual supports (eg symbols, timetables, Social Stories) if appropriate\n",
            "\n",
            "  * be aware of the environment (noisy/crowded) that you are in. Sensory differences may be affecting how much someone can process.  \n",
            "\n",
            "##  \n",
            "Avoiding open-ended questions  \n",
            "\n",
            "  * Keep questions short\n",
            "\n",
            "  * Ask only the most necessary questions\n",
            "\n",
            "  * Structure your questions, eg you could offer options or choices\n",
            "\n",
            "  * Be specific. For example, ask \"Did you enjoy your lunch?\" and \"Did you enjoy maths?\" rather than \"How was your day?\".  \n",
            "\n",
            "##  \n",
            "Ways to ask for help  \n",
            "\n",
            "If appropriate, give autistic people a visual help card to use to ask for\n",
            "help.  \n",
            "\n",
            "##  \n",
            "Being clear  \n",
            "\n",
            "Avoid using irony, sarcasm, figurative language, rhetorical questions, idioms\n",
            "or exaggeration as autistic people can take these literally. If you do use\n",
            "these, explain what you have said and be clear about what you really mean to\n",
            "say.  \n",
            "\n",
            "##  \n",
            "Distressed behaviour\n",
            "\n",
            "  * Use a behaviour diary to work out if the behaviour is a way of telling you something.  \n",
            "\n",
            "  * Offer other ways of expressing 'no' or 'stop'.  \n",
            "\n",
            "##  \n",
            "Reactions to \"no\"  \n",
            "\n",
            "  * Try using a different word or symbol.\n",
            "\n",
            "  * Autistic people may be confused about why you said no. If it's an activity that they can do later on that day or week, try showing this in a timetable.\n",
            "\n",
            "  * 'No' is often used when someone is putting themselves or others in danger. If it's a safety issue, look at ways of explaining danger and safety.\n",
            "\n",
            "  * If you are saying 'no' because someone is behaving inappropriately, you may want to change your reaction to their behaviour. Try not to shout or give too much attention, a calm reaction may help to decrease this behaviour in time.\n",
            "\n",
            "  * Set clear boundaries and explain why and where it is acceptable and not acceptable to behave in certain ways.  \n",
            "\n",
            "##  \n",
            "Next steps\n",
            "\n",
            "  * Read more about **behaviour**\n",
            "  * Find out about **visual supports**  and **social stories**\n",
            "  * **Find out more about autism**\n",
            "  * Find out about our **_Too Much Information_ campaign**.\n",
            "\n",
            "## Article sources\n",
            "\n",
            "Share guide:\n",
            "\n",
            "Last reviewed and updated on 20 August 2020\n",
            "\n",
            "### Was this page helpful?\n",
            "\n",
            "Yes No\n",
            "\n",
            "Read our privacy notice.  \n",
            "  \n",
            "Feedback is monitored periodically and will be considered when we update the\n",
            "page content. We cannot respond directly to feedback and it is not a way to\n",
            "access personal support. If you are seeking support, please visit our Help and\n",
            "support page.\n",
            "\n",
            "reCAPTCHA\n",
            "\n",
            "## Related guides\n",
            "\n",
            "## Social stories and comic strip conversations\n",
            "\n",
            "## Communication tools\n",
            "\n",
            "## Understanding and developing communication\n",
            "\n",
            "## Visual supports\n",
            "\n",
            "GW\n",
            "\n",
            "### You are not alone\n",
            "\n",
            "## Join the community\n",
            "\n",
            "Our online community is a great way to talk to like-minded people\n",
            "\n",
            "Visit the community\n",
            "\n",
            "### Keep up to date with all the great things we're doing\n",
            "\n",
            "Subscribe for updates\n",
            "\n",
            "### Useful Links\n",
            "\n",
            "  * Accessibility\n",
            "  * Contact us\n",
            "  * Privacy notice\n",
            "  * Terms and conditions \n",
            "  * Transparency and compliance\n",
            "\n",
            "  *   *   *   *   * \n",
            "\n",
            "We are registered as a charity in England and Wales (269425) and in Scotland\n",
            "(SC039427). The National Autistic Society is also a company limited by\n",
            "guarantee, registered at Companies House (01205298). VAT registration number:\n",
            "653370050. (C) The National Autistic Society 2024\n",
            "\n",
            "## Cookies on this site\n",
            "\n",
            "Our site uses cookies for key functions and to give you the best experience.\n",
            "Use cookie settings to control which cookies are allowed or click on Allow\n",
            "Optional Cookies to allow all cookies. For more detailed information please\n",
            "see our cookie policy.\n",
            "\n",
            "Choose cookie settings  Allow all cookies\n",
            "\n",
            "## Please wait while we check your current settings\n",
            "\n",
            "Your cookie settings\n",
            "\n",
            "Necessary cookies (always enabled)\n",
            "\n",
            "Functional cookies (help with key features, such as choices that you have\n",
            "made)\n",
            "\n",
            "Analytics cookies (helping us to understand how you use the site and therefore\n",
            "to improve it)\n",
            "\n",
            "Social and third-party cookies (where used)\n",
            "\n",
            "Choose cookie level Loading...\n",
            "\n",
            "\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: The provided URL contains tips for communicating effectively with autistic individuals. These tips include using their name, ensuring attention, using specific keywords, avoiding open-ended questions, providing visual support, being clear and avoiding figurative language, understanding distressed behavior, and setting clear boundaries. Additionally, the article suggests exploring further resources on behavior, visual supports, social stories, and autism in general.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "[DEBUG]: [Senior Web Research Analyst] Task output: The provided URL contains tips for communicating effectively with autistic individuals. These tips include using their name, ensuring attention, using specific keywords, avoiding open-ended questions, providing visual support, being clear and avoiding figurative language, understanding distressed behavior, and setting clear boundaries. Additionally, the article suggests exploring further resources on behavior, visual supports, social stories, and autism in general.\n",
            "\n",
            "\n",
            "\n",
            "[DEBUG]: Working Agent: Data Processing Specialist\n",
            "\n",
            "[INFO]: Starting Task: \n",
            "\n",
            " Transform the raw data collected by the Data Scraping Agent into a structured, modular format to support the creation of a detailed chatbot knowledge base. Each piece of data should be formatted as follows, focusing on singular, actionable strategies:\n",
            "{\n",
            "  ```\n",
            "  {\n",
            "  \"educational_strategy\": \"individual strategy\",\n",
            "  \"application\": \"detailed application in an educational or mentorship setting\"\n",
            "  }\n",
            "  ```.\n",
            "**Objective:**\n",
            "- Present each educational strategy and its application in a modular, concise format. This will serve as a foundational input for the Data Creation Agent, who is tasked with generating twenty unique prompts and responses for each strategy.\n",
            "\n",
            "**Guidelines:**\n",
            "1. Ensure each entry is focused on a single educational strategy, avoiding generalizations or combinations of strategies.\n",
            "2. Detail the application of each strategy in educational or mentorship settings, providing clear, actionable guidance.\n",
            "3. Prioritize clarity and specificity in describing how the strategy can be implemented, facilitating the Data Creation Agent in generating diverse and relevant responses.\n",
            "\n",
            "By adhering to these guidelines, you will contribute to building a comprehensive, structured knowledge base that enhances the chatbot’s ability to assist autistic students effectively.\n",
            "\n",
            "  \n",
            "\n",
            "\n",
            "\u001b[1m> Entering new CrewAgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? No\n",
            "Final Answer: ```\n",
            "{\n",
            "  \"educational_strategy\": \"Use their name\",\n",
            "  \"application\": \"Address autistic students by their name to establish personal connection and indicate that you are paying attention to them.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Ensure attention\",\n",
            "  \"application\": \"Gain and maintain an autistic student's attention before communicating, using eye contact, physical prompts, or their preferred attention-getting method.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Use specific keywords\",\n",
            "  \"application\": \"Incorporate keywords or phrases that are meaningful to the autistic student in communication to enhance understanding and engagement.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Avoid open-ended questions\",\n",
            "  \"application\": \"Frame questions in a specific and direct manner, avoiding open-ended inquiries that may be confusing or overwhelming for autistic students.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Provide visual support\",\n",
            "  \"application\": \"Utilize visual aids such as pictures, symbols, or diagrams to support communication, making abstract concepts more tangible and comprehensible.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Be clear and avoid figurative language\",\n",
            "  \"application\": \"Communicate using clear, concise language that is easy to understand, avoiding figurative language, sarcasm, or idioms that may be confusing.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Understand distressed behavior\",\n",
            "  \"application\": \"Recognize and understand the root causes of distressed behavior in autistic students, addressing the underlying issues rather than solely focusing on managing the behavior itself.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Set clear boundaries\",\n",
            "  \"application\": \"Establish and communicate clear boundaries and expectations for autistic students, providing a structured and predictable environment that fosters a sense of safety and security.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Explore further resources\",\n",
            "  \"application\": \"Seek additional information and resources on behavior, visual supports, social stories, and autism in general to enhance understanding and develop effective communication strategies.\"\n",
            "}\n",
            "```\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "[DEBUG]: [Data Processing Specialist] Task output: ```\n",
            "{\n",
            "  \"educational_strategy\": \"Use their name\",\n",
            "  \"application\": \"Address autistic students by their name to establish personal connection and indicate that you are paying attention to them.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Ensure attention\",\n",
            "  \"application\": \"Gain and maintain an autistic student's attention before communicating, using eye contact, physical prompts, or their preferred attention-getting method.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Use specific keywords\",\n",
            "  \"application\": \"Incorporate keywords or phrases that are meaningful to the autistic student in communication to enhance understanding and engagement.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Avoid open-ended questions\",\n",
            "  \"application\": \"Frame questions in a specific and direct manner, avoiding open-ended inquiries that may be confusing or overwhelming for autistic students.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Provide visual support\",\n",
            "  \"application\": \"Utilize visual aids such as pictures, symbols, or diagrams to support communication, making abstract concepts more tangible and comprehensible.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Be clear and avoid figurative language\",\n",
            "  \"application\": \"Communicate using clear, concise language that is easy to understand, avoiding figurative language, sarcasm, or idioms that may be confusing.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Understand distressed behavior\",\n",
            "  \"application\": \"Recognize and understand the root causes of distressed behavior in autistic students, addressing the underlying issues rather than solely focusing on managing the behavior itself.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Set clear boundaries\",\n",
            "  \"application\": \"Establish and communicate clear boundaries and expectations for autistic students, providing a structured and predictable environment that fosters a sense of safety and security.\"\n",
            "},\n",
            "{\n",
            "  \"educational_strategy\": \"Explore further resources\",\n",
            "  \"application\": \"Seek additional information and resources on behavior, visual supports, social stories, and autism in general to enhance understanding and develop effective communication strategies.\"\n",
            "}\n",
            "```\n",
            "\n",
            "\n",
            "\n",
            "[DEBUG]: Working Agent: Educational Content Developer\n",
            "\n",
            "[INFO]: Starting Task: \n",
            "  Objective: Construct a knowledge base for a chatbot that facilitates dialogues between autistic students and educators or mentors. This knowledge base is crucial for simulating realistic interactions focused on academic subjects, social interactions, and personal growth.\n",
            "\n",
            "  Knowledge base Entry Format:\n",
            "  ```\n",
            "  {\n",
            "  \"prompt\": \"text\",\n",
            "  \"response\": \"text\",\n",
            "  \"educational_strategy_used\": \"single strategy\",\n",
            "  \"application\": \"specific context\"\n",
            "  }\n",
            "  ```\n",
            "\n",
            "**Task Requirements:**\n",
            "- **Prompt Generation**: Create prompts that reflect authentic inquiries or comments from autistic students across a broad spectrum of topics. These prompts should embody the communication styles and needs unique to autistic students.\n",
            "\n",
            "- **Response Creation**: For each educational strategy provided by the Data Formatting Agent, you are tasked with generating twenty unique, supportive, and clear responses. Each response should be tailored to the student's query, utilizing the specified educational strategy in a way that is both supportive and personalized.\n",
            "\n",
            "- **Explicit Requirement**: It is imperative to generate a minimum of **TEN unique JSON objects for EACH educational strategy**. This means you will create twenty distinct prompts and corresponding responses for every single strategy provided, ensuring a diverse and comprehensive dataset.\n",
            "\n",
            "**Execution Strategy:**\n",
            "1. Identify various scenarios or questions an autistic student might have relating to each educational strategy.\n",
            "2. Develop twenty unique prompts for each strategy, ensuring a wide range of topics and situations are covered.\n",
            "3. Craft a tailored response for each prompt, applying the educational strategy in a supportive and context-specific manner.\n",
            "\n",
            "The goal of this task is to amass a rich, varied knowledge base that empowers the chatbot to offer realistic, educational, and supportive interactions for autistic students.\n",
            "Once agian to reinterate, remember to generate a minimum of **TEN unique JSON objects of prompt response pair for EACH educational strategy and application pair**\n",
            "   \n",
            "\n",
            "\n",
            "\u001b[1m> Entering new CrewAgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? No\n",
            "Final Answer: {\n",
            "  \"prompt\": \"I am feeling overwhelmed and confused with all the information in class. Can you help me understand it better?\",\n",
            "  \"response\": \"Hello [student's name], I can definitely help you with that. Let's break down the information into smaller, more manageable parts. Would you like me to use pictures or symbols to help you visualize the concepts?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I have a hard time understanding what the teacher is saying. Can you explain it to me in a different way?\",\n",
            "  \"response\": \"Sure, [student's name]. I can explain it in a different way. Would you like me to use pictures, diagrams, or symbols to help you visualize the concept?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm feeling nervous about the upcoming test. Can you help me prepare?\",\n",
            "  \"response\": \"Hello [student's name], I understand that you're feeling nervous about the upcoming test. Let's work together to create a study plan that suits your learning style. Would you like to use flashcards, practice tests, or mind maps to help you prepare?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm having trouble making friends at school. Can you give me some tips?\",\n",
            "  \"response\": \"Hi [student's name], I'm here to help you with that. Let's explore some strategies for making friends. Would you like to talk about your interests and hobbies? We can use that as a starting point to find common ground with other students.\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I feel like I'm always getting into trouble at school. Can you help me understand why?\",\n",
            "  \"response\": \"Hello [student's name], I'm here to help you understand why you might be getting into trouble at school. Let's work together to identify the root causes of the problem. Would you like to talk about your feelings and experiences? I'm here to listen.\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm having a hard time understanding the math concepts we're learning in class. Can you help me?\",\n",
            "  \"response\": \"Sure, [student's name]. I can help you with that. Let's break down the concepts into smaller, more manageable parts. Would you like me to use pictures, diagrams, or symbols to help you visualize the concepts?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm feeling overwhelmed with all the work I have to do. Can you help me prioritize my tasks?\",\n",
            "  \"response\": \"Hello [student's name], I understand that you're feeling overwhelmed. Let's work together to create a schedule that works for you. Would you like to use a planner or a digital tool to help you stay organized?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm having a hard time understanding the social cues in class. Can you help me?\",\n",
            "  \"response\": \"Sure, [student's name]. I can help you with that. Let's talk about different social cues and how to interpret them. Would you like to use role-playing or social stories to help you practice?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm feeling anxious about the upcoming presentation. Can you help me prepare?\",\n",
            "  \"response\": \"Hello [student's name], I understand that you're feeling anxious about the upcoming presentation. Let's work together to create a plan that will help you feel more confident. Would you like to practice your presentation in front of me or a small group?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm having trouble understanding the concepts in science class. Can you help me?\",\n",
            "  \"response\": \"Sure, [student's name]. I can help you with that. Let's break down the concepts into smaller, more manageable parts. Would you like me to use pictures, diagrams, or symbols to help you visualize the concepts?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "}\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "[DEBUG]: [Educational Content Developer] Task output: {\n",
            "  \"prompt\": \"I am feeling overwhelmed and confused with all the information in class. Can you help me understand it better?\",\n",
            "  \"response\": \"Hello [student's name], I can definitely help you with that. Let's break down the information into smaller, more manageable parts. Would you like me to use pictures or symbols to help you visualize the concepts?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I have a hard time understanding what the teacher is saying. Can you explain it to me in a different way?\",\n",
            "  \"response\": \"Sure, [student's name]. I can explain it in a different way. Would you like me to use pictures, diagrams, or symbols to help you visualize the concept?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm feeling nervous about the upcoming test. Can you help me prepare?\",\n",
            "  \"response\": \"Hello [student's name], I understand that you're feeling nervous about the upcoming test. Let's work together to create a study plan that suits your learning style. Would you like to use flashcards, practice tests, or mind maps to help you prepare?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm having trouble making friends at school. Can you give me some tips?\",\n",
            "  \"response\": \"Hi [student's name], I'm here to help you with that. Let's explore some strategies for making friends. Would you like to talk about your interests and hobbies? We can use that as a starting point to find common ground with other students.\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I feel like I'm always getting into trouble at school. Can you help me understand why?\",\n",
            "  \"response\": \"Hello [student's name], I'm here to help you understand why you might be getting into trouble at school. Let's work together to identify the root causes of the problem. Would you like to talk about your feelings and experiences? I'm here to listen.\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm having a hard time understanding the math concepts we're learning in class. Can you help me?\",\n",
            "  \"response\": \"Sure, [student's name]. I can help you with that. Let's break down the concepts into smaller, more manageable parts. Would you like me to use pictures, diagrams, or symbols to help you visualize the concepts?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm feeling overwhelmed with all the work I have to do. Can you help me prioritize my tasks?\",\n",
            "  \"response\": \"Hello [student's name], I understand that you're feeling overwhelmed. Let's work together to create a schedule that works for you. Would you like to use a planner or a digital tool to help you stay organized?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm having a hard time understanding the social cues in class. Can you help me?\",\n",
            "  \"response\": \"Sure, [student's name]. I can help you with that. Let's talk about different social cues and how to interpret them. Would you like to use role-playing or social stories to help you practice?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm feeling anxious about the upcoming presentation. Can you help me prepare?\",\n",
            "  \"response\": \"Hello [student's name], I understand that you're feeling anxious about the upcoming presentation. Let's work together to create a plan that will help you feel more confident. Would you like to practice your presentation in front of me or a small group?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "},\n",
            "{\n",
            "  \"prompt\": \"I'm having trouble understanding the concepts in science class. Can you help me?\",\n",
            "  \"response\": \"Sure, [student's name]. I can help you with that. Let's break down the concepts into smaller, more manageable parts. Would you like me to use pictures, diagrams, or symbols to help you visualize the concepts?\",\n",
            "  \"educational_strategy_used\": \"Use their name\",\n",
            "  \"application\": \"Establishing a personal connection and indicating attention to the student by addressing them by their name.\"\n",
            "}\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Define your agents outside the loop as their definitions do not change per link\n",
        "\n",
        "# TODO :- create education content extraction agent and task\n",
        "\n",
        "data_scraping_agent = Agent(\n",
        "  role='Senior Web Research Analyst',\n",
        "  goal='To meticulously scrape and gather data from various web pages, focusing on educational strategies and communication techniques for teaching and mentoring autistic students.',\n",
        "  backstory=\"\"\"\n",
        "    As a Senior Web Research Analyst, your journey has been marked by a deep commitment to supporting the educational needs of autistic students through the power of information technology.\n",
        "    With a passion for uncovering the most effective teaching strategies and communication techniques,\n",
        "    you have become an expert in navigating the vast expanses of the internet to find those gems of knowledge hidden within.\n",
        "    Your dedication stems from a personal mission to bridge the gap between educational research and practical application,\n",
        "    ensuring that every piece of data you extract has the potential to transform the learning experience of autistic students.\n",
        "    Your expertise not only lies in the technical skills required to scrape web data but also in your ability to discern the relevance and impact of this information on real-world educational practices.\n",
        "    Over the years, your work has contributed to building a more inclusive and understanding educational environment for autistic students, one webpage at a time.\n",
        "\n",
        "   \"\"\",\n",
        "  verbose=True,\n",
        "  allow_delegation=False,\n",
        "  llm = llm,  # using Google Gemini Pro API\n",
        "  tools=[\n",
        "        html_to_text\n",
        "      ]\n",
        ")\n",
        "\n",
        "\n",
        "\n",
        "data_formatting_agent = Agent(\n",
        "  role='Data Processing Specialist',\n",
        "  goal='To structure and format the diverse data gathered from the web, focusing on educational strategies and communication techniques for autistic students.',\n",
        "  backstory=\"\"\"\n",
        "    As a Data Processing Specialist with a profound interest in the nexus of data science and special education,\n",
        "    you have carved out a niche for yourself by transforming raw,\n",
        "     unstructured data into meticulously organized knowledge that can directly impact the educational experiences of autistic students.\n",
        "     Your journey has been characterized by a relentless pursuit of clarity and utility in data,\n",
        "     driven by the belief that the right information, when properly formatted,\n",
        "     can unlock unprecedented educational opportunities. With an eye for detail and a heart for inclusivity,\n",
        "     you specialize in distilling complex data into actionable insights that empower educators, mentors, and students alike.\n",
        "     Your work is more than just about organizing information;\n",
        "     it's about creating pathways to understanding and engagement for autistic students through the power of structured knowledge.\n",
        "     The chatbot knowledge repository you're helping to build stands as a testament to your dedication to making education accessible and meaningful for all learners,\n",
        "     regardless of their neurological makeup.\n",
        "\n",
        "  \"\"\",\n",
        "  verbose=True,\n",
        "  allow_delegation=False,\n",
        "  llm = llm,  # using Google Gemini Pro API\n",
        "  tools=[]\n",
        ")\n",
        "\n",
        "\n",
        "\n",
        "data_creation_agent = Agent(\n",
        "  role='Educational Content Developer',\n",
        "  goal='To create educational and mentorship materials from the structured data, focusing on strategies and communication techniques for autistic students.',\n",
        "  backstory=\"\"\"\n",
        "\n",
        "    In your role as an Educational Content Developer, you have honed a unique ability to transform structured data into engaging,\n",
        "    interactive learning experiences for autistic students. With a deep understanding of the nuances of educational strategies and the importance of personalized communication,\n",
        "    you are at the forefront of developing content that makes a real difference. Your journey has been driven by a commitment to enhance the educational landscape for autistic students,\n",
        "    ensuring that each piece of content not only educates but also supports their personal and social development.\n",
        "    Through your work, you aim to bridge the gap between data and practical application,\n",
        "    creating a knowledge repository that serves as a lifeline for educators and mentors striving to connect with and support their autistic students.\n",
        "    Your expertise is not just in content creation but in envisioning how each interaction can foster understanding, growth, and confidence in autistic learners.\n",
        "    The materials you develop are more than just educational tools; they are pillars of support for those navigating the complexities of learning and interaction within the autism spectrum.\n",
        "\n",
        "  \"\"\",\n",
        "  verbose=True,\n",
        "  allow_delegation=False,\n",
        "  llm = llm,  # using Google Gemini Pro API\n",
        "  tools=[]\n",
        ")\n",
        "\n",
        "# List of links to process\n",
        "links = [\n",
        "     \"https://www.autism.org.uk/advice-and-guidance/topics/communication/understanding-and-developing-communication\",\n",
        "     \"https://www.autism.org.uk/advice-and-guidance/what-is-autism/varying-support-needs\",\n",
        "     \"https://www.autism.org.uk/advice-and-guidance/topics/communication/tips\",\n",
        "     \"https://www.autism.org.uk/advice-and-guidance/stories/stories-from-the-spectrum-patrick-samuel\",\n",
        "     \"https://www.autism.org.uk/advice-and-guidance/topics/communication/communication-tools/visual-supports\",\n",
        "     \"https://www.autism.org.uk/what-we-do/campaign/education\"\n",
        "]\n",
        "\n",
        "# Results dictionary to store results for each link\n",
        "results = {}\n",
        "\n",
        "\n",
        "# Task for Data Formatting Agent\n",
        "data_formatting_agent_task = Task(\n",
        "  description=\"\"\"\n",
        "\n",
        " Transform the raw data collected by the Data Scraping Agent into a structured, modular format to support the creation of a detailed chatbot knowledge base. Each piece of data should be formatted as follows, focusing on singular, actionable strategies:\n",
        "{\n",
        "  ```\n",
        "  {\n",
        "  \"educational_strategy\": \"individual strategy\",\n",
        "  \"application\": \"detailed application in an educational or mentorship setting\"\n",
        "  }\n",
        "  ```.\n",
        "**Objective:**\n",
        "- Present each educational strategy and its application in a modular, concise format. This will serve as a foundational input for the Data Creation Agent, who is tasked with generating twenty unique prompts and responses for each strategy.\n",
        "\n",
        "**Guidelines:**\n",
        "1. Ensure each entry is focused on a single educational strategy, avoiding generalizations or combinations of strategies.\n",
        "2. Detail the application of each strategy in educational or mentorship settings, providing clear, actionable guidance.\n",
        "3. Prioritize clarity and specificity in describing how the strategy can be implemented, facilitating the Data Creation Agent in generating diverse and relevant responses.\n",
        "\n",
        "By adhering to these guidelines, you will contribute to building a comprehensive, structured knowledge base that enhances the chatbot’s ability to assist autistic students effectively.\n",
        "\n",
        "  \"\"\",\n",
        "  agent= data_formatting_agent\n",
        ")\n",
        "\n",
        "# Create tasks for your agents\n",
        "\n",
        "data_creation_agent_task = Task(\n",
        "  description=\"\"\"\n",
        "  Objective: Construct a knowledge base for a chatbot that facilitates dialogues between autistic students and educators or mentors. This knowledge base is crucial for simulating realistic interactions focused on academic subjects, social interactions, and personal growth.\n",
        "\n",
        "  Knowledge base Entry Format:\n",
        "  ```\n",
        "  {\n",
        "  \"prompt\": \"text\",\n",
        "  \"response\": \"text\",\n",
        "  \"educational_strategy_used\": \"single strategy\",\n",
        "  \"application\": \"specific context\"\n",
        "  }\n",
        "  ```\n",
        "\n",
        "**Task Requirements:**\n",
        "- **Prompt Generation**: Create prompts that reflect authentic inquiries or comments from autistic students across a broad spectrum of topics. These prompts should embody the communication styles and needs unique to autistic students.\n",
        "\n",
        "- **Response Creation**: For each educational strategy provided by the Data Formatting Agent, you are tasked with generating twenty unique, supportive, and clear responses. Each response should be tailored to the student's query, utilizing the specified educational strategy in a way that is both supportive and personalized.\n",
        "\n",
        "- **Explicit Requirement**: It is imperative to generate a minimum of **TEN unique JSON objects for EACH educational strategy**. This means you will create twenty distinct prompts and corresponding responses for every single strategy provided, ensuring a diverse and comprehensive dataset.\n",
        "\n",
        "**Execution Strategy:**\n",
        "1. Identify various scenarios or questions an autistic student might have relating to each educational strategy.\n",
        "2. Develop twenty unique prompts for each strategy, ensuring a wide range of topics and situations are covered.\n",
        "3. Craft a tailored response for each prompt, applying the educational strategy in a supportive and context-specific manner.\n",
        "\n",
        "The goal of this task is to amass a rich, varied knowledge base that empowers the chatbot to offer realistic, educational, and supportive interactions for autistic students.\n",
        "Once agian to reinterate, remember to generate a minimum of **TEN unique JSON objects of prompt response pair for EACH educational strategy and application pair**\n",
        "   \"\"\",\n",
        "  agent=data_creation_agent\n",
        ")\n",
        "\n",
        "\n",
        "for link in links:\n",
        "    # Update task for Data Scraping Agent with the current link\n",
        "    data_scraping_agent_task = Task(\n",
        "        description=f\"\"\"\n",
        "        Your task is to meticulously search and scrape web pages for content related to both teaching strategies and communication techniques for autistic students.\n",
        "        Focus on identifying articles, studies, and resources that offer insights into effective educational practices and mentorship guidance.\n",
        "        Use the html_to_text tool to convert the content of the following URL to plain text: {link}\n",
        "        Extract key points that offer insights into both educational strategies and effective ways to communicate and mentor autistic students, ensuring the content's relevance and accuracy.\"\"\",\n",
        "        agent=data_scraping_agent\n",
        "    )\n",
        "\n",
        "\n",
        "    # Define the crew with the updated task\n",
        "    crew = Crew(\n",
        "        agents=[data_scraping_agent, data_formatting_agent, data_creation_agent],\n",
        "        tasks=[data_scraping_agent_task, data_formatting_agent_task, data_creation_agent_task],\n",
        "        verbose=2,  # Verbose level set to 2 for detailed logging\n",
        "        process=Process.sequential,\n",
        "    )\n",
        "\n",
        "    # Kickoff the crew for the current link and store the result\n",
        "    result = crew.kickoff()\n",
        "    results[link] = result\n",
        "\n",
        "# After the loop, you have a dictionary of results for each link\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}