13.llm-output-format.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "ZWXtHsfR1CRx"
   },
   "outputs": [],
   "source": [
    "!pip -qqq install pip --progress-bar off\n",
    "!pip -qqq install groq==0.9.0 --progress-bar off\n",
    "!pip -qqq install pydantic==2.7.4 --progress-bar off\n",
    "!pip -qqq install langchain-core==0.2.7 --progress-bar off"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "import json\n",
    "import os\n",
    "from enum import Enum\n",
    "from typing import Generic, Type, TypeVar\n",
    "\n",
    "import groq\n",
    "from google.colab import userdata\n",
    "from groq import Groq\n",
    "from langchain_core.output_parsers import PydanticOutputParser\n",
    "from pydantic import BaseModel\n",
    "\n",
    "os.environ[\"GROQ_API_KEY\"] = userdata.get(\"GROQ_API_KEY\")\n",
    "MODEL = \"llama3-70b-8192\""
   ],
   "metadata": {
    "id": "DaIV6agC2zYt"
   },
   "execution_count": 2,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "tweet = \"\"\"\n",
    "How do you choose which LLM to use?\n",
    "\n",
    "A vibe check ain't going to cut it.\n",
    "\n",
    "I'm trying DeepEval (@jeffr_yyy)\n",
    "\n",
    "- Wide range of metrics: Relevancy, Hallucination & more\n",
    "- Bulk & real-time evaluation\n",
    "- CI/CD integration\n",
    "- Benchmarking on popular datasets\n",
    "\"\"\""
   ],
   "metadata": {
    "id": "gcGRpt7O2-5m"
   },
   "execution_count": 3,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "client = Groq(api_key=os.environ.get(\"GROQ_API_KEY\"))\n",
    "\n",
    "\n",
    "class ResponseFormat(Enum):\n",
    "    JSON = \"json\"\n",
    "    TEXT = \"text\"\n",
    "\n",
    "\n",
    "def predict(\n",
    "    prompt: str,\n",
    "    system_prompt: str = None,\n",
    "    response_format: ResponseFormat = ResponseFormat.TEXT,\n",
    "    model: str = MODEL,\n",
    "    client: Groq = client,\n",
    "):\n",
    "    messages = [\n",
    "        {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": prompt,\n",
    "        }\n",
    "    ]\n",
    "    if system_prompt:\n",
    "        messages.insert(\n",
    "            0,\n",
    "            {\n",
    "                \"role\": \"system\",\n",
    "                \"content\": system_prompt,\n",
    "            },\n",
    "        )\n",
    "    try:\n",
    "        chat_completion = client.chat.completions.create(\n",
    "            messages=messages,\n",
    "            model=model,\n",
    "            temperature=0.00001,\n",
    "            response_format={\n",
    "                \"type\": \"json_object\"\n",
    "                if response_format == ResponseFormat.JSON\n",
    "                else \"text\"\n",
    "            },\n",
    "        )\n",
    "        return chat_completion.choices[0].message.content\n",
    "    except groq.APIConnectionError as e:\n",
    "        print(\"The server could not be reached\")\n",
    "        print(e.__cause__)\n",
    "    except groq.RateLimitError as e:\n",
    "        print(\"A 429 status code was received; we should back off a bit.\")\n",
    "    except groq.APIStatusError as e:\n",
    "        print(\"Another non-200-range status code was received\")\n",
    "        print(e.status_code)\n",
    "        print(e.message)\n",
    "        print(e.response)"
   ],
   "metadata": {
    "id": "FAOrKrTX9qi4"
   },
   "execution_count": 4,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## JSON Output Support"
   ],
   "metadata": {
    "id": "n6OoU_-peTnk"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "system_prompt = \"\"\"\n",
    "You're evaluating writing style in text.\n",
    "Your evalutions must always be in JSON format. Here is an example JSON response:\n",
    "\n",
    "```\n",
    "{\n",
    "  'readability': 4,\n",
    "  'conciseness': 2\n",
    "}\n",
    "```\n",
    "\"\"\"\n",
    "\n",
    "print(system_prompt)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fbGU_Fp5ms_o",
    "outputId": "2302d7e0-9715-410e-b357-e21da0e7517c"
   },
   "execution_count": 5,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "You're evaluating writing style in text.\n",
      "Your evalutions must always be in JSON format. Here is an example JSON response:\n",
      "\n",
      "```\n",
      "{\n",
      "  'readability': 4,\n",
      "  'conciseness': 2\n",
      "}\n",
      "```\n",
      "\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "prompt = f\"\"\"\n",
    "Evaluate the text:\n",
    "\n",
    "```\n",
    "{tweet}\n",
    "```\n",
    "\n",
    "You're evaluating the readability and conciseness with values from 0 (extremely bad) to 10 (extremely good)\n",
    "\"\"\"\n",
    "print(prompt)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZtT6HRlXmp2f",
    "outputId": "d9591c47-24a0-4709-fe9e-a492b5819526"
   },
   "execution_count": 6,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Evaluate the text:\n",
      "\n",
      "```\n",
      "\n",
      "How do you choose which LLM to use?\n",
      "\n",
      "A vibe check ain't going to cut it.\n",
      "\n",
      "I'm trying DeepEval (@jeffr_yyy)\n",
      "\n",
      "- Wide range of metrics: Relevancy, Hallucination & more\n",
      "- Bulk & real-time evaluation\n",
      "- CI/CD integration\n",
      "- Benchmarking on popular datasets\n",
      "\n",
      "```\n",
      "\n",
      "You're evaluating the readability and conciseness with values from 0 (extremely bad) to 10 (extremely good)\n",
      "\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "response = predict(prompt, system_prompt, response_format=ResponseFormat.JSON)"
   ],
   "metadata": {
    "id": "_IVKObyGn7pp"
   },
   "execution_count": 7,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "print(response)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qugrxq7zosln",
    "outputId": "8875a6c7-04d7-4b0c-93ad-09d78c50cfd7"
   },
   "execution_count": 8,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "{\n",
      "\"readability\": 8,\n",
      "\"conciseness\": 6\n",
      "}\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## No JSON Output Support"
   ],
   "metadata": {
    "id": "hn8JyJmFeV8D"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "class WritingScore(BaseModel):\n",
    "    readability: int\n",
    "    conciseness: int"
   ],
   "metadata": {
    "id": "NE8b7_AFvXSi"
   },
   "execution_count": 9,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "schema = {k: v for k, v in WritingScore.schema().items()}\n",
    "schema = {\"properties\": schema[\"properties\"], \"required\": schema[\"required\"]}\n",
    "print(json.dumps(schema, indent=2))"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Lsf8Ub_qvYcv",
    "outputId": "3a76acf5-db1e-4813-d680-e9194de6f8d6"
   },
   "execution_count": 10,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "{\n",
      "  \"properties\": {\n",
      "    \"readability\": {\n",
      "      \"title\": \"Readability\",\n",
      "      \"type\": \"integer\"\n",
      "    },\n",
      "    \"conciseness\": {\n",
      "      \"title\": \"Conciseness\",\n",
      "      \"type\": \"integer\"\n",
      "    }\n",
      "  },\n",
      "  \"required\": [\n",
      "    \"readability\",\n",
      "    \"conciseness\"\n",
      "  ]\n",
      "}\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "OUTPUT_FORMAT_INSTRUCTIONS = \"\"\"The output should be formatted as a JSON instance that conforms to the JSON schema below.\n",
    "\n",
    "As an example, for the schema {{\"properties\": {{\"foo\": {{\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {{\"type\": \"string\"}}}}}}, \"required\": [\"foo\"]}}\n",
    "the object {{\"foo\": [\"bar\", \"baz\"]}} is a well-formatted instance of the schema. The object {{\"properties\": {{\"foo\": [\"bar\", \"baz\"]}}}} is not well-formatted.\n",
    "\n",
    "Here is the output schema:\n",
    "\n",
    "```\n",
    "{schema}\n",
    "```\n",
    "\n",
    "Do not return any preamble or explanations, return only a pure JSON string surrounded by triple backticks (```).\"\"\""
   ],
   "metadata": {
    "id": "68jzcw8fvbGO"
   },
   "execution_count": 11,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "json_instruction = OUTPUT_FORMAT_INSTRUCTIONS.format(\n",
    "    schema=json.dumps(schema, indent=2)\n",
    ")\n",
    "print(json_instruction)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WhW-ohIJvcWo",
    "outputId": "7318a3b3-082a-41d5-ed43-0cffbb727120"
   },
   "execution_count": 12,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The output should be formatted as a JSON instance that conforms to the JSON schema below.\n",
      "\n",
      "As an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\n",
      "the object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n",
      "\n",
      "Here is the output schema:\n",
      "\n",
      "```\n",
      "{\n",
      "  \"properties\": {\n",
      "    \"readability\": {\n",
      "      \"title\": \"Readability\",\n",
      "      \"type\": \"integer\"\n",
      "    },\n",
      "    \"conciseness\": {\n",
      "      \"title\": \"Conciseness\",\n",
      "      \"type\": \"integer\"\n",
      "    }\n",
      "  },\n",
      "  \"required\": [\n",
      "    \"readability\",\n",
      "    \"conciseness\"\n",
      "  ]\n",
      "}\n",
      "```\n",
      "\n",
      "Do not return any preamble or explanations, return only a pure JSON string surrounded by triple backticks (```).\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "prompt = f\"\"\"\n",
    "Evaluate the writing style from the text:\n",
    "\n",
    "```\n",
    "{tweet}\n",
    "```\n",
    "\n",
    "Evaluate the readability and conciseness with values from 0 (extremely bad) to 10 (extremely good)\n",
    "\n",
    "{json_instruction}\n",
    "\"\"\"\n",
    "print(prompt)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GAhiXP5-vmaw",
    "outputId": "d135d4d0-bd5b-41fc-deb8-40054fc7233b"
   },
   "execution_count": 13,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "Evaluate the writing style from the text:\n",
      "\n",
      "```\n",
      "\n",
      "How do you choose which LLM to use?\n",
      "\n",
      "A vibe check ain't going to cut it.\n",
      "\n",
      "I'm trying DeepEval (@jeffr_yyy)\n",
      "\n",
      "- Wide range of metrics: Relevancy, Hallucination & more\n",
      "- Bulk & real-time evaluation\n",
      "- CI/CD integration\n",
      "- Benchmarking on popular datasets\n",
      "\n",
      "```\n",
      "\n",
      "Evaluate the readability and conciseness with values from 0 (extremely bad) to 10 (extremely good)\n",
      "\n",
      "The output should be formatted as a JSON instance that conforms to the JSON schema below.\n",
      "\n",
      "As an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\n",
      "the object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n",
      "\n",
      "Here is the output schema:\n",
      "\n",
      "```\n",
      "{\n",
      "  \"properties\": {\n",
      "    \"readability\": {\n",
      "      \"title\": \"Readability\",\n",
      "      \"type\": \"integer\"\n",
      "    },\n",
      "    \"conciseness\": {\n",
      "      \"title\": \"Conciseness\",\n",
      "      \"type\": \"integer\"\n",
      "    }\n",
      "  },\n",
      "  \"required\": [\n",
      "    \"readability\",\n",
      "    \"conciseness\"\n",
      "  ]\n",
      "}\n",
      "```\n",
      "\n",
      "Do not return any preamble or explanations, return only a pure JSON string surrounded by triple backticks (```).\n",
      "\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "response = predict(prompt, MODEL)\n",
    "print(response)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "veYpcsEFvpbt",
    "outputId": "8dc41cf5-4312-475d-c960-5dd4623dd1fd"
   },
   "execution_count": 14,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "```\n",
      "{\n",
      "  \"readability\": 8,\n",
      "  \"conciseness\": 9\n",
      "}\n",
      "```\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## From Scratch"
   ],
   "metadata": {
    "id": "3NWTNINsFRXU"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "response_json = json.loads(response.strip(\"```\"))\n",
    "WritingScore.parse_obj(response_json)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "A-kAhaLeNRbD",
    "outputId": "2c25363b-830f-42a5-f654-08df9ec23471"
   },
   "execution_count": 15,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "WritingScore(readability=8, conciseness=9)"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "TBaseModel = TypeVar(\"TBaseModel\", bound=BaseModel)\n",
    "\n",
    "\n",
    "class JsonOutputParser(Generic[TBaseModel]):\n",
    "    def __init__(self, pydantic_object: Type[TBaseModel]):\n",
    "        self.pydantic_object = pydantic_object\n",
    "\n",
    "    def parse(self, response: str):\n",
    "        response_json = json.loads(response.strip(\"```\"))\n",
    "        return self.pydantic_object.parse_obj(response_json)"
   ],
   "metadata": {
    "id": "rOwcuyScP-YQ"
   },
   "execution_count": 16,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "JsonOutputParser(pydantic_object=WritingScore).parse(response)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xnyYTE0mXss8",
    "outputId": "7fd45560-28b8-45f1-e721-4675b7d64808"
   },
   "execution_count": 17,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "WritingScore(readability=8, conciseness=9)"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Using LangChain Parser"
   ],
   "metadata": {
    "id": "0KSkb7XwOgYL"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "parser = PydanticOutputParser(pydantic_object=WritingScore)"
   ],
   "metadata": {
    "id": "9kVcsoQEOitm"
   },
   "execution_count": 18,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "parser.parse(response)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "25Gqy9rvvBoE",
    "outputId": "ed769c3e-ed3b-4f44-f6dd-509cd7cf97ed"
   },
   "execution_count": 19,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "WritingScore(readability=8, conciseness=9)"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ]
  }
 ]
}