diff --git a/README.rst b/README.rst index 8f4e762..08d7307 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,10 @@ OpenAI Cost Logger ================== -Simple cost logger for OpenAI requests. -Track the cost of every request you make to OpenAI and visualize them in a user-friendly way. +* Simple **cost logger** for **OpenAI requests**. +* Track the cost of every request you make to OpenAI and visualize them in a user-friendly way. +* Homepage on `PyPI `_. +* `Demo file `_ with a usage example. How to install: --------------- @@ -16,13 +18,10 @@ How to install: * .. code-block:: python - from openai_cost_logger.constants import DEFAULT_LOG_PATH, Models, MODELS_COST - from openai_cost_logger.openai_cost_logger_viz import OpenAICostLoggerViz - from openai_cost_logger.openai_cost_logger_utils import OpenAICostLoggerUtils - from openai_cost_logger.openai_cost_logger import OpenAICostLogger - -* See also the homepage on `PyPI `_. -* See the `demo file `_ for a usage example. + from openai_cost_logger import OpenAICostLogger + from openai_cost_logger import OpenAICostLoggerViz + from openai_cost_logger import OpenAICostLoggerUtils + from openai_cost_logger import DEFAULT_LOG_PATH, MODELS_COST Key Features: ------------- @@ -30,22 +29,17 @@ Key Features: * Choose the feature you want to track (prompt_tokens, completion_tokens, completion, prompt, etc.). * Check the cost of your requests filtering by model or strftime aggregation (see the docs). -Endpoint supported: +Models supported: ------------------- -* Chat completion. -* Every response passed to *OpenAICostLogger* should contain the fields "*usage.prompt_tokens*" and "*usage.completion_tokens*". - This is the only strict requirement of the library, the way you call the OpenAI API is totally up to you. If needed, you can - find an easy example in the demo file. +* The response generation is totally up to the user. The library support every model which response contains the fields **usage.prompt_tokens** and **usage.total_tokens** (e.g. chat completions, embeddings, etc.). -Viz examples: -------------- -.. image::images/viz_prints.png - :alt: Viz prints examples. - :align: center - :width: 500px +Note: +----- +* Every cost is specified per **million tokens**. +* If you don't specify the cost, the library will look to the **MODELS_COST** dictionary and get the cost of the model you are using. Be aware that if the model is not in the dictionary, an exception will be raised. -.. image::images/strftime_agg.png - :alt: Strftime aggregation example. +Viz example: +------------- +.. image:: images/example.png + :alt: Viz example (prints + plot) :align: center - :width: 500px - diff --git a/changes_proposal.md b/changes_proposal.md index dd34342..80dbfd4 100644 --- a/changes_proposal.md +++ b/changes_proposal.md @@ -1,4 +1,4 @@ -1. ⌛ model has to be provided in form of enum - important, hard to juggle with all 0xxx versions +1. ✅ model has to be provided in form of enum - important, hard to juggle with all 0xxx versions - Merged Change: - we can just infer it from `response.model` @@ -15,7 +15,7 @@ Motivation: - allows easier integration, user only has to initialize tracker object and call `update_cost(response)`, otherwise each chat completion call would have to be rewritten -4. ✅ log file just acumulates total cost +4. ✅ log file just acumulates total cost - Merged Change: - add breakdown of responses/input token per response/output token per response/cost per response @@ -45,4 +45,6 @@ Change: - change strftime format to `strftime("%Y-%m-%d_%H:%M:%S")`, makes it more readable - we could possibly infer the datetime and do plots with datetime instead of str -6. WIP +6. ⌛ web ui for stats viz + +7. WIP diff --git a/demo.ipynb b/demo.ipynb index 09af674..0d330ee 100644 --- a/demo.ipynb +++ b/demo.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 28, + "execution_count": 76, "metadata": {}, "outputs": [ { @@ -28,97 +28,96 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 79, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Defaulting to user installation because normal site-packages is not writeable\n", "Requirement already satisfied: openai==1.13.3 in /home/drudao/.local/lib/python3.10/site-packages (from -r requirements.txt (line 1)) (1.13.3)\n", "Requirement already satisfied: matplotlib==3.6.3 in /home/drudao/.local/lib/python3.10/site-packages (from -r requirements.txt (line 2)) (3.6.3)\n", "Requirement already satisfied: pytest==7.4.2 in /home/drudao/.local/lib/python3.10/site-packages (from -r requirements.txt (line 3)) (7.4.2)\n", - "Requirement already satisfied: tqdm>4 in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (4.66.2)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai==1.13.3->-r requirements.txt (line 1)) (1.7.0)\n", - "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (2.6.4)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (0.27.0)\n", "Requirement already satisfied: anyio<5,>=3.5.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (4.3.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (0.27.0)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (2.6.4)\n", "Requirement already satisfied: sniffio in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (1.3.1)\n", + "Requirement already satisfied: tqdm>4 in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (4.66.2)\n", "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/drudao/.local/lib/python3.10/site-packages (from openai==1.13.3->-r requirements.txt (line 1)) (4.10.0)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (4.42.1)\n", - "Requirement already satisfied: cycler>=0.10 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (0.11.0)\n", - "Requirement already satisfied: packaging>=20.0 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (23.1)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (2.8.2)\n", "Requirement already satisfied: contourpy>=1.0.1 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (1.1.1)\n", + "Requirement already satisfied: cycler>=0.10 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (0.11.0)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (4.42.1)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (1.4.5)\n", - "Requirement already satisfied: pillow>=6.2.0 in /usr/lib/python3/dist-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (9.0.1)\n", "Requirement already satisfied: numpy>=1.19 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (1.26.4)\n", - "Requirement already satisfied: pyparsing>=2.2.1 in /usr/lib/python3/dist-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (2.4.7)\n", + "Requirement already satisfied: packaging>=20.0 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (23.1)\n", + "Requirement already satisfied: pillow>=6.2.0 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (10.2.0)\n", + "Requirement already satisfied: pyparsing>=2.2.1 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib==3.6.3->-r requirements.txt (line 2)) (2.8.2)\n", "Requirement already satisfied: iniconfig in /home/drudao/.local/lib/python3.10/site-packages (from pytest==7.4.2->-r requirements.txt (line 3)) (2.0.0)\n", "Requirement already satisfied: pluggy<2.0,>=0.12 in /home/drudao/.local/lib/python3.10/site-packages (from pytest==7.4.2->-r requirements.txt (line 3)) (1.3.0)\n", "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /home/drudao/.local/lib/python3.10/site-packages (from pytest==7.4.2->-r requirements.txt (line 3)) (1.1.3)\n", "Requirement already satisfied: tomli>=1.0.0 in /home/drudao/.local/lib/python3.10/site-packages (from pytest==7.4.2->-r requirements.txt (line 3)) (2.0.1)\n", - "Requirement already satisfied: idna>=2.8 in /usr/lib/python3/dist-packages (from anyio<5,>=3.5.0->openai==1.13.3->-r requirements.txt (line 1)) (3.3)\n", - "Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from httpx<1,>=0.23.0->openai==1.13.3->-r requirements.txt (line 1)) (2020.6.20)\n", + "Requirement already satisfied: idna>=2.8 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai==1.13.3->-r requirements.txt (line 1)) (3.6)\n", + "Requirement already satisfied: certifi in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai==1.13.3->-r requirements.txt (line 1)) (2024.2.2)\n", "Requirement already satisfied: httpcore==1.* in /home/drudao/.local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai==1.13.3->-r requirements.txt (line 1)) (1.0.4)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /home/drudao/.local/lib/python3.10/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai==1.13.3->-r requirements.txt (line 1)) (0.14.0)\n", "Requirement already satisfied: annotated-types>=0.4.0 in /home/drudao/.local/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai==1.13.3->-r requirements.txt (line 1)) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.16.3 in /home/drudao/.local/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai==1.13.3->-r requirements.txt (line 1)) (2.16.3)\n", - "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib==3.6.3->-r requirements.txt (line 2)) (1.16.0)\n" + "Requirement already satisfied: six>=1.5 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib==3.6.3->-r requirements.txt (line 2)) (1.16.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ - "!pip install -r requirements.txt" + "%pip install -r requirements.txt" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 80, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: openai_cost_logger in /home/drudao/.local/lib/python3.10/site-packages (0.0.2)\n", - "Requirement already satisfied: openai in /home/drudao/.local/lib/python3.10/site-packages (from openai_cost_logger) (1.13.3)\n", - "Requirement already satisfied: pandas in /home/drudao/.local/lib/python3.10/site-packages (from openai_cost_logger) (2.1.1)\n", - "Requirement already satisfied: matplotlib in /home/drudao/.local/lib/python3.10/site-packages (from openai_cost_logger) (3.6.3)\n", - "Requirement already satisfied: numpy>=1.19 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai_cost_logger) (1.26.4)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai_cost_logger) (1.1.1)\n", - "Requirement already satisfied: cycler>=0.10 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai_cost_logger) (0.11.0)\n", - "Requirement already satisfied: pillow>=6.2.0 in /usr/lib/python3/dist-packages (from matplotlib->openai_cost_logger) (9.0.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai_cost_logger) (4.42.1)\n", - "Requirement already satisfied: pyparsing>=2.2.1 in /usr/lib/python3/dist-packages (from matplotlib->openai_cost_logger) (2.4.7)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai_cost_logger) (2.8.2)\n", - "Requirement already satisfied: packaging>=20.0 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai_cost_logger) (23.1)\n", - "Requirement already satisfied: kiwisolver>=1.0.1 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai_cost_logger) (1.4.5)\n", - "Requirement already satisfied: sniffio in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai_cost_logger) (1.3.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai_cost_logger) (4.10.0)\n", - "Requirement already satisfied: tqdm>4 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai_cost_logger) (4.66.2)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai->openai_cost_logger) (1.7.0)\n", - "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai_cost_logger) (2.6.4)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai_cost_logger) (0.27.0)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai_cost_logger) (4.3.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/lib/python3/dist-packages (from pandas->openai_cost_logger) (2022.1)\n", - "Requirement already satisfied: tzdata>=2022.1 in /home/drudao/.local/lib/python3.10/site-packages (from pandas->openai_cost_logger) (2023.3)\n", - "Requirement already satisfied: idna>=2.8 in /usr/lib/python3/dist-packages (from anyio<5,>=3.5.0->openai->openai_cost_logger) (3.3)\n", - "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/drudao/.local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai->openai_cost_logger) (1.1.3)\n", - "Requirement already satisfied: httpcore==1.* in /home/drudao/.local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai->openai_cost_logger) (1.0.4)\n", - "Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from httpx<1,>=0.23.0->openai->openai_cost_logger) (2020.6.20)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /home/drudao/.local/lib/python3.10/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai->openai_cost_logger) (0.14.0)\n", - "Requirement already satisfied: annotated-types>=0.4.0 in /home/drudao/.local/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai->openai_cost_logger) (0.6.0)\n", - "Requirement already satisfied: pydantic-core==2.16.3 in /home/drudao/.local/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai->openai_cost_logger) (2.16.3)\n", - "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib->openai_cost_logger) (1.16.0)\n", + "Requirement already satisfied: openai-cost-logger in /home/drudao/.local/lib/python3.10/site-packages (0.0.2)\n", + "Requirement already satisfied: matplotlib in /home/drudao/.local/lib/python3.10/site-packages (from openai-cost-logger) (3.6.3)\n", + "Requirement already satisfied: openai in /home/drudao/.local/lib/python3.10/site-packages (from openai-cost-logger) (1.13.3)\n", + "Requirement already satisfied: pandas in /home/drudao/.local/lib/python3.10/site-packages (from openai-cost-logger) (2.1.1)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (1.1.1)\n", + "Requirement already satisfied: cycler>=0.10 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (0.11.0)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (4.42.1)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (1.4.5)\n", + "Requirement already satisfied: numpy>=1.19 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (1.26.4)\n", + "Requirement already satisfied: packaging>=20.0 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (23.1)\n", + "Requirement already satisfied: pillow>=6.2.0 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (10.2.0)\n", + "Requirement already satisfied: pyparsing>=2.2.1 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /home/drudao/.local/lib/python3.10/site-packages (from matplotlib->openai-cost-logger) (2.8.2)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai-cost-logger) (4.3.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from openai->openai-cost-logger) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai-cost-logger) (0.27.0)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai-cost-logger) (2.6.4)\n", + "Requirement already satisfied: sniffio in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai-cost-logger) (1.3.1)\n", + "Requirement already satisfied: tqdm>4 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai-cost-logger) (4.66.2)\n", + "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/drudao/.local/lib/python3.10/site-packages (from openai->openai-cost-logger) (4.10.0)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from pandas->openai-cost-logger) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.1 in /home/drudao/.local/lib/python3.10/site-packages (from pandas->openai-cost-logger) (2023.3)\n", + "Requirement already satisfied: idna>=2.8 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai->openai-cost-logger) (3.6)\n", + "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/drudao/.local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai->openai-cost-logger) (1.1.3)\n", + "Requirement already satisfied: certifi in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai->openai-cost-logger) (2024.2.2)\n", + "Requirement already satisfied: httpcore==1.* in /home/drudao/.local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai->openai-cost-logger) (1.0.4)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /home/drudao/.local/lib/python3.10/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai->openai-cost-logger) (0.14.0)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /home/drudao/.local/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai->openai-cost-logger) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.16.3 in /home/drudao/.local/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai->openai-cost-logger) (2.16.3)\n", + "Requirement already satisfied: six>=1.5 in /home/drudao/anaconda3/envs/modern_nlp/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib->openai-cost-logger) (1.16.0)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ - "pip install openai_cost_logger" + "%pip install openai-cost-logger" ] }, { @@ -130,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -145,14 +144,22 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 67, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "imported openai_cost_logger\n" + ] + } + ], "source": [ - "from openai_cost_logger.constants import DEFAULT_LOG_PATH, Models, MODELS_COST\n", - "from openai_cost_logger.openai_cost_logger_viz import OpenAICostLoggerViz\n", - "from openai_cost_logger.openai_cost_logger_utils import OpenAICostLoggerUtils\n", - "from openai_cost_logger.openai_cost_logger import OpenAICostLogger" + "from openai_cost_logger import OpenAICostLogger\n", + "from openai_cost_logger import OpenAICostLoggerViz\n", + "from openai_cost_logger import OpenAICostLoggerUtils\n", + "from openai_cost_logger import DEFAULT_LOG_PATH, MODELS_COST" ] }, { @@ -164,18 +171,15 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 68, "metadata": {}, "outputs": [], "source": [ "# Export the proper environment variables based on the client you are using.\n", "\n", "# OpenAI API Key\n", - "os.environ[\"OPENAI_ORGANIZATION\"] = OpenAICostLoggerUtils.get_api_key(path='openai_organization.txt')\n", - "os.environ[\"OPENAI_API_KEY\"] = OpenAICostLoggerUtils.get_api_key(path='openai_api_key.txt')\n", - "\n", - "# Azure OpenAI API Key\n", - "os.environ[\"AZURE_OPENAI_KEY\"] = OpenAICostLoggerUtils.get_api_key('azure_openai_key.txt')" + "os.environ[\"OPENAI_API_KEY\"] = OpenAICostLoggerUtils.read_api_key(path='openai_api_key.txt')\n", + "os.environ[\"OPENAI_ORGANIZATION\"] = OpenAICostLoggerUtils.read_api_key(path='openai_organization.txt')" ] }, { @@ -187,92 +191,90 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ + "# The following demo is based on the standard OpenAI API client but it is easily adaptable for every client and model\n", + "# since the response generation is totally decoupled from the logging process (and is totally up to the user).\n", + "client = openai.OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n", "experiment_name = \"Demo\"\n", - "messages = [\n", - " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"Who won the euro 2020?\"},\n", - "]\n", "cost_upperbound = 2\n", "log_folder = DEFAULT_LOG_PATH" ] }, { - "cell_type": "code", - "execution_count": 35, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# Azure OpenAI usage\n", - "model = Models.AZURE_3_5_TURBO\n", - "client_args = {\n", - " \"azure_endpoint\": \"https://your_key.openai.azure.com/\",\n", - " \"api_key\": os.getenv(\"AZURE_OPENAI_KEY\"),\n", - " \"api_version\": \"your_api_version\",\n", - "}\n", - "input_cost = MODELS_COST[model.value][\"input\"]\n", - "output_cost = MODELS_COST[model.value][\"output\"]" + "### 5. Demo" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ - "# OpenAI usage\n", - "client = openai.OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n", - "model = Models.TURBO_3_5\n", - "client_args = {}\n", - "input_cost = MODELS_COST[model.value][\"input\"]\n", - "output_cost = MODELS_COST[model.value][\"output\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5. Demo" + "# Instantiate the OpenAICostLogger\n", + "cost_logger = OpenAICostLogger(\n", + " experiment_name = experiment_name,\n", + " log_folder = log_folder,\n", + " cost_upperbound = cost_upperbound\n", + ")" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 71, "metadata": {}, + "outputs": [], "source": [ - "**NOTE:**\n", + "# CHAT COMPLETION EXAMPLE\n", + "model = \"gpt-3.5-turbo\"\n", + "input_cost = MODELS_COST[model][\"input\"]\n", + "output_cost = MODELS_COST[model][\"output\"]\n", + "messages = [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\"role\": \"user\", \"content\": \"Who won the euro 2020?\"},\n", + "]\n", + "\n", + "# Get the model response\n", + "response = client.chat.completions.create(model=model, messages=messages, max_tokens=1, temperature=0)\n", "\n", - "The logger is independent of the OpenAI api call. Indeed, It only require the endpoint answer as input and the user is fully responsible of the model call. Despite that, in the cells below you can find a full working demo." + "# In case `input_cost` or `output_cost` are not passed, the object will look for the model in the `MODELS_COST` dictionary.\n", + "# If the model is not found, it will raise an exception.\n", + "# The costs should be per million tokens.\n", + "cost_logger.update_cost(response=response, input_cost=input_cost, output_cost=output_cost)" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ - "# Create the OpenAICostLogger object\n", - "cost_logger = OpenAICostLogger(\n", - " experiment_name = experiment_name,\n", - " model = model.value,\n", - " input_cost = input_cost,\n", - " output_cost = output_cost,\n", - " log_folder = log_folder,\n", - " cost_upperbound = cost_upperbound\n", - ")" + "# EMBEDDINGS CREATION EXAMPLE\n", + "model = \"text-embedding-ada-002\"\n", + "input_cost = MODELS_COST[model][\"input\"]\n", + "output_cost = MODELS_COST[model][\"output\"]\n", + "messages = [\"Once upon a time\", \"There was a frog\"]\n", + "\n", + "# In case `input_cost` or `output_cost` are not passed, the object will look for the model in the `MODELS_COST` dictionary.\n", + "# If the model is not found, it will raise an exception.\n", + "# The costs should be per million tokens.\n", + "response = client.embeddings.create(model=model, input=[\"Once upon a time\", \"There was a frog\"])\n", + "cost_logger.update_cost(response=response, input_cost=input_cost, output_cost=output_cost)" ] }, { - "cell_type": "code", - "execution_count": 38, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# Run the chat completion endpoint\n", - "response = client.chat.completions.create(model=model.value, messages=messages, max_tokens=1, temperature=0)\n", - "cost_logger.update_cost(response)" + "**NOTE**:\\\n", + "The above examples show the usage using `chat completion` and `embedding`. However, all the API endpoints are supported.\\\n", + "The only strict requirements is that the model response contains the fields `usage.total_tokens` and `usage.prompt_tokens`.\\\n", + "Be also aware that the `content` of the response is logged only for the `chat completion`." ] }, { @@ -284,49 +286,49 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 73, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Total cost: 0.000986 (USD)\n" + "Total cost: 3e-05 (USD)\n" ] } ], "source": [ "# Print the total cost\n", - "OpenAICostLoggerViz.print_total_cost(path=DEFAULT_LOG_PATH)" + "OpenAICostLoggerViz.print_total_cost(path=log_folder)" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 77, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "gpt-3.5-turbo: 0.000887 (USD)\n", - "gpt-35-turbo-0125: 9.9e-05 (USD)\n" + "gpt-3.5-turbo-0125: 2.8e-05 (USD)\n", + "text-embedding-ada-002: 2e-06 (USD)\n" ] } ], "source": [ "# Cost by model\n", - "OpenAICostLoggerViz.print_total_cost_by_model(path=DEFAULT_LOG_PATH)" + "OpenAICostLoggerViz.print_total_cost_by_model(path=log_folder)" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 75, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAHHCAYAAACfqw0dAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8bUlEQVR4nO3de1hVZd7/8Q8HOXgANJWDoqB5zJJRE9HKShIVC8qe0KEwNbUDZZn5U/OQnWisHPNQjFOpmY6FU1Zq9BimlTKomOUBzRTNpsADAooKyr5/f/i4Z3aiwgrdgO/Xde2Li3t9172+a7eKT2vfLFyMMUYAAACoEFdnNwAAAFAdEaIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAOAi5s+fLxcXF23atMkpx7/11lt16623OuXYAC6OEAXAqfbs2aORI0eqRYsW8vLyko+Pj3r06KE33nhDJ0+erPTjnThxQs8995zWrFlT6XMDuLq4O7sBAFevFStW6H/+53/k6emphIQEdejQQSUlJfr222/1zDPPaPv27Zo7d26lHvPEiROaOnWqJHGHB8AfQogC4BTZ2dkaOHCgmjdvrtWrVyswMNC+7bHHHtNPP/2kFStWOLFDALg4Ps4D4BTTpk3T8ePH9c477zgEqHOuvfZajRo1yv79mTNn9MILL6hly5by9PRUSEiIJkyYoOLiYof9Nm3apKioKDVs2FDe3t4KDQ3V0KFDJUn79u1To0aNJElTp06Vi4uLXFxc9Nxzz12y3xMnTmjkyJG65ppr5OPjo4SEBB09etS+ffDgwWrYsKFOnz593r69e/dWmzZtLnmMuXPnqmXLlvL29lbXrl31zTffnFdTUlKiyZMnq3PnzvL19VWdOnV0880366uvvrLXGGMUEhKimJiY8/Y/deqUfH19NXLkyEv2A+DiCFEAnOKzzz5TixYt1L1793LVP/TQQ5o8ebI6deqkv/71r+rZs6eSkpI0cOBAe83BgwfVu3dv7du3T+PGjdOsWbMUHx+vf/3rX5KkRo0a6a233pIk3X333Vq4cKEWLlyoe+6555LHT0xMVFZWlp577jklJCRo0aJFio2NlTFGkvTAAw/oyJEj+uKLLxz2y8nJ0erVq3X//fdfdP533nlHI0eOVEBAgKZNm6YePXrorrvu0oEDBxzqCgsL9fbbb+vWW2/VX/7yFz333HM6dOiQoqKitGXLFkmSi4uL7r//fn3++efKy8tz2P+zzz5TYWHhJfsBUA4GAK6wgoICI8nExMSUq37Lli1GknnooYccxseMGWMkmdWrVxtjjPn444+NJLNx48YLznXo0CEjyUyZMqVcx543b56RZDp37mxKSkrs49OmTTOSzCeffGKMMaa0tNQ0bdrUxMXFOew/ffp04+LiYvbu3XvBY5SUlJjGjRubsLAwU1xcbB+fO3eukWR69uxpHztz5oxDjTHGHD161Pj7+5uhQ4fax3bt2mUkmbfeesuh9q677jIhISHGZrOV6/wBXBh3ogBccYWFhZKkevXqlat+5cqVkqTRo0c7jD/99NOSZF875efnJ0lavnx5mR+r/REjRoxQrVq17N8/8sgjcnd3t/fm6uqq+Ph4ffrppzp27Ji9btGiRerevbtCQ0MvOPemTZt08OBBPfzww/Lw8LCPP/jgg/L19XWodXNzs9fYbDbl5eXpzJkz6tKlizZv3myva926tcLDw7Vo0SL7WF5enj7//HPFx8fLxcXF4jsB4BxCFIArzsfHR5IcwsbF7N+/X66urrr22msdxgMCAuTn56f9+/dLknr27KkBAwZo6tSpatiwoWJiYjRv3rzz1k1Z0apVK4fv69atq8DAQO3bt88+lpCQoJMnT+rjjz+WJO3atUuZmZl64IEHLnl+ZR2jVq1aatGixXn1CxYs0A033CAvLy9dc801atSokVasWKGCggKHuoSEBK1bt84+f0pKik6fPn3JfgCUDyEKwBXn4+OjoKAgbdu2rUL7XeruiYuLi5YuXar09HQlJibq3//+t4YOHarOnTvr+PHjf6Tlcmnfvr06d+6s999/X5L0/vvvy8PDQ/fdd1+lHeP999/Xgw8+qJYtW+qdd95RamqqVq1apdtvv102m82hduDAgapVq5b9btT777+vLl26lGuRO4BLI0QBcIr+/ftrz549Sk9Pv2Rt8+bNZbPZtHv3bofx3Nxc5efnq3nz5g7j3bp100svvaRNmzZp0aJF2r59u5YsWSLp0kHsQn5/7OPHj+u3335TSEiIw3hCQoJWr16t3377TYsXL1Z0dLTq169/yfMr6xinT59Wdna2w9jSpUvVokULffTRR3rggQcUFRWlyMhInTp16rx5GzRooOjoaC1atEj79+/XunXruAsFVCJCFACnGDt2rOrUqaOHHnpIubm5523fs2eP3njjDUlSv379JEkzZsxwqJk+fbokKTo6WpJ09OhR+2/LnRMWFiZJ9o/0ateuLUnKz8+vUL9z5851WGf11ltv6cyZM+rbt69D3aBBg+Ti4qJRo0Zp79695fotuC5duqhRo0ZKTk5WSUmJfXz+/Pnn9enm5iZJDueZkZFxwTD6wAMPaMeOHXrmmWfk5ubm8NuMAP4YHrYJwClatmypxYsXKy4uTu3atXN4Yvn69euVkpKiBx98UJLUsWNHDR48WHPnzlV+fr569uypDRs2aMGCBYqNjdVtt90m6exaoTfffFN33323WrZsqWPHjunvf/+7fHx87EHM29tb7du31wcffKDWrVurQYMG6tChgzp06HDRfktKStSrVy/dd9992rVrl958803ddNNNuuuuuxzqGjVqpD59+iglJUV+fn72gHcxtWrV0osvvqiRI0fq9ttvV1xcnLKzszVv3rzz1kT1799fH330ke6++25FR0crOztbycnJat++fZkfWUZHR+uaa65RSkqK+vbtq8aNG1+yHwDl5OxfDwRwdfvxxx/N8OHDTUhIiPHw8DD16tUzPXr0MLNmzTKnTp2y150+fdpMnTrVhIaGmlq1apng4GAzfvx4h5rNmzebQYMGmWbNmhlPT0/TuHFj079/f7Np0yaHY65fv9507tzZeHh4XPJxB+cecbB27VozYsQIU79+fVO3bl0THx9vjhw5UuY+H374oZFkRowYUaH34s033zShoaHG09PTdOnSxXz99demZ8+eDo84sNls5uWXXzbNmzc3np6e5k9/+pNZvny5GTx4sGnevHmZ8z766KNGklm8eHGF+gFwcS7G/O7eNwDgD/nkk08UGxurr7/+WjfffLOz29FTTz2ld955Rzk5OfaPMwH8cYQoAKhk/fv3V1ZWln766SenP4/p1KlTCg4OVv/+/TVv3jyn9gLUNKyJAoBKsmTJEv3www9asWKF3njjDacGqIMHD+rLL7/U0qVLdeTIEYe/QwigcnAnCgAqiYuLi+rWrau4uDglJyfL3d15/5+6Zs0a3XbbbWrcuLEmTZqkxMREp/UC1FSEKAAAAAt4ThQAAIAFhCgAAAALWFh+GdlsNv3666+qV6+e039DBwAAlI8xRseOHVNQUJBcXS98v4kQdRn9+uuvCg4OdnYbAADAggMHDqhp06YX3E6Iuozq1asn6ew/BB8fHyd3AwAAyqOwsFDBwcH2n+MXQoi6jM59hOfj40OIAgCgmrnUUhwWlgMAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFrg7uwEAAGqakHErnN3CVWHfK9FOPT53ogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFTg9Rc+bMUUhIiLy8vBQeHq4NGzZctD4lJUVt27aVl5eXrr/+eq1cudJhuzFGkydPVmBgoLy9vRUZGandu3c71OTl5Sk+Pl4+Pj7y8/PTsGHDdPz4cYeaL774Qt26dVO9evXUqFEjDRgwQPv27auUcwYAANWfU0PUBx98oNGjR2vKlCnavHmzOnbsqKioKB08eLDM+vXr12vQoEEaNmyYvvvuO8XGxio2Nlbbtm2z10ybNk0zZ85UcnKyMjIyVKdOHUVFRenUqVP2mvj4eG3fvl2rVq3S8uXL9fXXX2vEiBH27dnZ2YqJidHtt9+uLVu26IsvvtDhw4d1zz33XL43AwAAVCsuxhjjrIOHh4frxhtv1OzZsyVJNptNwcHBevzxxzVu3Ljz6uPi4lRUVKTly5fbx7p166awsDAlJyfLGKOgoCA9/fTTGjNmjCSpoKBA/v7+mj9/vgYOHKisrCy1b99eGzduVJcuXSRJqamp6tevn3755RcFBQVp6dKlGjRokIqLi+XqejZnfvbZZ4qJiVFxcbFq1apVrvMrLCyUr6+vCgoK5OPj84feKwBA9REyboWzW7gq7Hsl+rLMW96f3067E1VSUqLMzExFRkb+pxlXV0VGRio9Pb3MfdLT0x3qJSkqKspen52drZycHIcaX19fhYeH22vS09Pl5+dnD1CSFBkZKVdXV2VkZEiSOnfuLFdXV82bN0+lpaUqKCjQwoULFRkZedEAVVxcrMLCQocXAAComZwWog4fPqzS0lL5+/s7jPv7+ysnJ6fMfXJyci5af+7rpWoaN27ssN3d3V0NGjSw14SGhup///d/NWHCBHl6esrPz0+//PKLPvzww4ueU1JSknx9fe2v4ODgi9YDAIDqy+kLy6uinJwcDR8+XIMHD9bGjRu1du1aeXh46N5779XFPv0cP368CgoK7K8DBw5cwa4BAMCV5O6sAzds2FBubm7Kzc11GM/NzVVAQECZ+wQEBFy0/tzX3NxcBQYGOtSEhYXZa36/cP3MmTPKy8uz7z9nzhz5+vpq2rRp9pr3339fwcHBysjIULdu3crsz9PTU56enpc6dQAAUAM47U6Uh4eHOnfurLS0NPuYzWZTWlqaIiIiytwnIiLCoV6SVq1aZa8PDQ1VQECAQ01hYaEyMjLsNREREcrPz1dmZqa9ZvXq1bLZbAoPD5cknThxwr6g/Bw3Nzd7jwAAAE79OG/06NH6+9//rgULFigrK0uPPPKIioqKNGTIEElSQkKCxo8fb68fNWqUUlNT9frrr2vnzp167rnntGnTJiUmJkqSXFxc9OSTT+rFF1/Up59+qq1btyohIUFBQUGKjY2VJLVr1059+vTR8OHDtWHDBq1bt06JiYkaOHCggoKCJEnR0dHauHGjnn/+ee3evVubN2/WkCFD1Lx5c/3pT3+6sm8SAACokpz2cZ509pEFhw4d0uTJk5WTk6OwsDClpqbaF4b//PPPDneEunfvrsWLF2vixImaMGGCWrVqpWXLlqlDhw72mrFjx6qoqEgjRoxQfn6+brrpJqWmpsrLy8tes2jRIiUmJqpXr15ydXXVgAEDNHPmTPv222+/XYsXL9a0adM0bdo01a5dWxEREUpNTZW3t/cVeGcAAEBV59TnRNV0PCcKAK5OPCfqyrhqnxMFAABQnRGiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwwOkhas6cOQoJCZGXl5fCw8O1YcOGi9anpKSobdu28vLy0vXXX6+VK1c6bDfGaPLkyQoMDJS3t7ciIyO1e/duh5q8vDzFx8fLx8dHfn5+GjZsmI4fP37ePK+99ppat24tT09PNWnSRC+99FLlnDQAAKj2nBqiPvjgA40ePVpTpkzR5s2b1bFjR0VFRengwYNl1q9fv16DBg3SsGHD9N133yk2NlaxsbHatm2bvWbatGmaOXOmkpOTlZGRoTp16igqKkqnTp2y18THx2v79u1atWqVli9frq+//lojRoxwONaoUaP09ttv67XXXtPOnTv16aefqmvXrpfnjQAAANWOizHGOOvg4eHhuvHGGzV79mxJks1mU3BwsB5//HGNGzfuvPq4uDgVFRVp+fLl9rFu3bopLCxMycnJMsYoKChITz/9tMaMGSNJKigokL+/v+bPn6+BAwcqKytL7du318aNG9WlSxdJUmpqqvr166dffvlFQUFBysrK0g033KBt27apTZs2ls+vsLBQvr6+KigokI+Pj+V5AADVS8i4Fc5u4aqw75XoyzJveX9+O+1OVElJiTIzMxUZGfmfZlxdFRkZqfT09DL3SU9Pd6iXpKioKHt9dna2cnJyHGp8fX0VHh5ur0lPT5efn589QElSZGSkXF1dlZGRIUn67LPP1KJFCy1fvlyhoaEKCQnRQw89pLy8vIueU3FxsQoLCx1eAACgZnJaiDp8+LBKS0vl7+/vMO7v76+cnJwy98nJyblo/bmvl6pp3Lixw3Z3d3c1aNDAXrN3717t379fKSkpeu+99zR//nxlZmbq3nvvveg5JSUlydfX1/4KDg6+aD0AAKi+nL6wvCqy2WwqLi7We++9p5tvvlm33nqr3nnnHX311VfatWvXBfcbP368CgoK7K8DBw5cwa4BAMCV5LQQ1bBhQ7m5uSk3N9dhPDc3VwEBAWXuExAQcNH6c18vVfP7hetnzpxRXl6evSYwMFDu7u5q3bq1vaZdu3aSpJ9//vmC5+Tp6SkfHx+HFwAAqJmcFqI8PDzUuXNnpaWl2cdsNpvS0tIUERFR5j4REREO9ZK0atUqe31oaKgCAgIcagoLC5WRkWGviYiIUH5+vjIzM+01q1evls1mU3h4uCSpR48eOnPmjPbs2WOv+fHHHyVJzZs3/yOnDQAAagh3Zx589OjRGjx4sLp06aKuXbtqxowZKioq0pAhQyRJCQkJatKkiZKSkiSdfexAz5499frrrys6OlpLlizRpk2bNHfuXEmSi4uLnnzySb344otq1aqVQkNDNWnSJAUFBSk2NlbS2TtKffr00fDhw5WcnKzTp08rMTFRAwcOVFBQkKSzC807deqkoUOHasaMGbLZbHrsscd0xx13ONydAgAAVy+nhqi4uDgdOnRIkydPVk5OjsLCwpSammpfGP7zzz/L1fU/N8u6d++uxYsXa+LEiZowYYJatWqlZcuWqUOHDvaasWPHqqioSCNGjFB+fr5uuukmpaamysvLy16zaNEiJSYmqlevXnJ1ddWAAQM0c+ZM+3ZXV1d99tlnevzxx3XLLbeoTp066tu3r15//fUr8K4AAIDqwKnPiarpeE4UAFydeE7UlXHVPicKAACgOiNEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAvfyFDVo0KBCk7q4uGjz5s1q3ry5paYAAACqunKFqPz8fM2YMUO+vr6XrDXG6NFHH1Vpaekfbg4AAKCqKleIkqSBAweqcePG5ap9/PHHLTcEAABQHZQrRNlstgpNeuzYMUvNAAAAVBcsLAcAALDgD4eorKwszZs3T1u2bKmEdgAAAKqHcq+JkqTnn39e3t7eeuaZZyRJX331lfr06aN69eqpoKBA8+fPV3x8/GVpFAAAoCqp0J2opUuXqn379vbvX3rpJT3xxBM6fPiwZs+erZdffrnSGwQAAKiKynUn6r333pMxRvv27dOWLVt05MgRGWO0bt063XzzzXrvvfdks9m0d+9evffee5KkhISEy9o4AACAM5UrRJ17aKaHh4f8/f3VvHlzbdmyRT4+PrrttttkjFFxcbFcXFwUEhIiY8xlbRoAAMDZyhWievbsKUnq1KmTli9frv/3//6fUlNT1a9fP91yyy2SpK1btyo4ONj+PQAAQE1WoTVRr776qrZs2aIePXpo//79ev755+3b5s+frz59+lR6gwAAAFVRhX47r2PHjtq3b5+OHDmia665xmHbmDFj5OPjU6nNAQAAVFUVClHn/D5ASVJgYOAfbgYAAKC6KNfHeTNnztSpU6fKPWlycjJ/+gUAANRo5QpRTz31VIVC0dixY3Xo0CHLTQEAAFR15fo4zxijXr16yd29fJ/+nTx58g81BQAAUNWVKxVNmTKlQpPGxMSoQYMGlhoCAACoDi5LiAIAAKjpKvScKAAAAJxFiAIAALCAEAUAAGABIQoAAMCCCoeo559/XidOnDhv/OTJkw5/Sw8AAKAmq3CImjp1qo4fP37e+IkTJzR16tRKaQoAAKCqq3CIMsbIxcXlvPHvv/+eZ0MBAICrRrn/AHH9+vXl4uIiFxcXtW7d2iFIlZaW6vjx43r44YcvS5MAAABVTblD1IwZM2SM0dChQzV16lT5+vrat3l4eCgkJEQRERGXpUkAAICqptwhavDgwZKk0NBQ9ejRo9x/Rw8AAKAmqvCaqHr16ikrK8v+/SeffKLY2FhNmDBBJSUlldocAABAVVXhEDVy5Ej9+OOPkqS9e/cqLi5OtWvXVkpKisaOHVvpDQIAAFRFFQ5RP/74o8LCwiRJKSkp6tmzpxYvXqz58+frn//8Z2X3BwAAUCVZesSBzWaTJH355Zfq16+fJCk4OFiHDx+u3O4AAACqqAqHqC5duujFF1/UwoULtXbtWkVHR0uSsrOz5e/vX+kNAgAAVEUVDlEzZszQ5s2blZiYqGeffVbXXnutJGnp0qXq3r17pTcIAABQFVX4OQU33HCDtm7det74q6++Kjc3t0ppCgAAoKqz/LCnzMxM+6MO2rdvr06dOlVaUwAAAFVdhUPUwYMHFRcXp7Vr18rPz0+SlJ+fr9tuu01LlixRo0aNKrtHAACAKqfCa6Ief/xxHT9+XNu3b1deXp7y8vK0bds2FRYW6oknnrgcPQIAAFQ5Fb4TlZqaqi+//FLt2rWzj7Vv315z5sxR7969K7U5AACAqqrCd6JsNptq1ap13nitWrXsz48CAACo6Socom6//XaNGjVKv/76q33s3//+t5566in16tWrUpsDAACoqiocombPnq3CwkKFhISoZcuWatmypUJDQ1VYWKhZs2Zdjh4BAACqnAqviQoODtbmzZv15ZdfaufOnZKkdu3aKTIystKbAwAAqKosPSfKxcVFd9xxh+64447K7gcAAKBaKPfHeatXr1b79u1VWFh43raCggJdd911+uabbyq1OQAAgKqq3CFqxowZGj58uHx8fM7b5uvrq5EjR2r69OmV2hwAAEBVVe4Q9f3336tPnz4X3N67d29lZmZaamLOnDkKCQmRl5eXwsPDtWHDhovWp6SkqG3btvLy8tL111+vlStXOmw3xmjy5MkKDAyUt7e3IiMjtXv3boeavLw8xcfHy8fHR35+fho2bJiOHz9e5vF++ukn1atXz/6EdgAAgHKHqNzc3DKfD3WOu7u7Dh06VOEGPvjgA40ePVpTpkzR5s2b1bFjR0VFRengwYNl1q9fv16DBg3SsGHD9N133yk2NlaxsbHatm2bvWbatGmaOXOmkpOTlZGRoTp16igqKkqnTp2y18THx2v79u1atWqVli9frq+//lojRow473inT5/WoEGDdPPNN1f43AAAQM1V7hDVpEkTh6Dyez/88IMCAwMr3MD06dM1fPhwDRkyRO3bt1dycrJq166td999t8z6N954Q3369NEzzzyjdu3a6YUXXlCnTp00e/ZsSWfvQs2YMUMTJ05UTEyMbrjhBr333nv69ddftWzZMklSVlaWUlNT9fbbbys8PFw33XSTZs2apSVLljg8/0qSJk6cqLZt2+q+++6r8LkBAICaq9whql+/fpo0aZLD3ZxzTp48qSlTpqh///4VOnhJSYkyMzMdHo/g6uqqyMhIpaenl7lPenr6eY9TiIqKstdnZ2crJyfHocbX11fh4eH2mvT0dPn5+alLly72msjISLm6uiojI8M+tnr1aqWkpGjOnDnlOp/i4mIVFhY6vAAAQM1U7kccTJw4UR999JFat26txMREtWnTRpK0c+dOzZkzR6WlpXr22WcrdPDDhw+rtLRU/v7+DuP+/v72Z1D9Xk5OTpn1OTk59u3nxi5W07hxY4ft7u7uatCggb3myJEjevDBB/X++++XuZi+LElJSZo6dWq5agEAQPVW7hDl7++v9evX65FHHtH48eNljJF09plRUVFRmjNnznnBpTobPny4/vznP+uWW24p9z7jx4/X6NGj7d8XFhYqODj4crQHAACcrEIP22zevLlWrlypo0eP6qeffpIxRq1atVL9+vUtHbxhw4Zyc3NTbm6uw3hubq4CAgLK3CcgIOCi9ee+5ubmOqzRys3NVVhYmL3m9wvXz5w5o7y8PPv+q1ev1qeffqrXXntN0tm1VjabTe7u7po7d66GDh16Xm+enp7y9PQs7+kDAIBqrMJ/O0+S6tevrxtvvFFdu3a1HKAkycPDQ507d1ZaWpp9zGazKS0tTREREWXuExER4VAvSatWrbLXh4aGKiAgwKGmsLBQGRkZ9pqIiAjl5+c7PJJh9erVstlsCg8Pl3R23dSWLVvsr+eff1716tXTli1bdPfdd1s+ZwAAUDNY+rMvlWn06NEaPHiwunTpoq5du2rGjBkqKirSkCFDJEkJCQlq0qSJkpKSJEmjRo1Sz5499frrrys6OlpLlizRpk2bNHfuXElnP1588skn9eKLL6pVq1YKDQ3VpEmTFBQUpNjYWEln/9Zfnz59NHz4cCUnJ+v06dNKTEzUwIEDFRQUZK/5b5s2bZKrq6s6dOhwhd4ZAABQlTk9RMXFxenQoUOaPHmycnJyFBYWptTUVPv6qp9//lmurv+5Yda9e3ctXrxYEydO1IQJE9SqVSstW7bMIdyMHTtWRUVFGjFihPLz83XTTTcpNTVVXl5e9ppFixYpMTFRvXr1kqurqwYMGKCZM2deuRMHAADVmos5t0Icla6wsFC+vr4qKCgo92/4AQCqv5BxK5zdwlVh3yvRl2Xe8v78trQmCgAA4GpHiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwIIqEaLmzJmjkJAQeXl5KTw8XBs2bLhofUpKitq2bSsvLy9df/31WrlypcN2Y4wmT56swMBAeXt7KzIyUrt373aoycvLU3x8vHx8fOTn56dhw4bp+PHj9u1r1qxRTEyMAgMDVadOHYWFhWnRokWVd9IAAKBac3qI+uCDDzR69GhNmTJFmzdvVseOHRUVFaWDBw+WWb9+/XoNGjRIw4YN03fffafY2FjFxsZq27Zt9ppp06Zp5syZSk5OVkZGhurUqaOoqCidOnXKXhMfH6/t27dr1apVWr58ub7++muNGDHC4Tg33HCD/vnPf+qHH37QkCFDlJCQoOXLl1++NwMAAFQbLsYY48wGwsPDdeONN2r27NmSJJvNpuDgYD3++OMaN27cefVxcXEqKipyCDPdunVTWFiYkpOTZYxRUFCQnn76aY0ZM0aSVFBQIH9/f82fP18DBw5UVlaW2rdvr40bN6pLly6SpNTUVPXr10+//PKLgoKCyuw1Ojpa/v7+evfdd8t1boWFhfL19VVBQYF8fHwq9L4AAKqvkHErnN3CVWHfK9GXZd7y/vx26p2okpISZWZmKjIy0j7m6uqqyMhIpaenl7lPenq6Q70kRUVF2euzs7OVk5PjUOPr66vw8HB7TXp6uvz8/OwBSpIiIyPl6uqqjIyMC/ZbUFCgBg0aVPxEAQBAjePuzIMfPnxYpaWl8vf3dxj39/fXzp07y9wnJyenzPqcnBz79nNjF6tp3Lixw3Z3d3c1aNDAXvN7H374oTZu3Ki//e1vFzyf4uJiFRcX278vLCy8YC0AAKjenL4mqjr46quvNGTIEP3973/Xddddd8G6pKQk+fr62l/BwcFXsEsAAHAlOTVENWzYUG5ubsrNzXUYz83NVUBAQJn7BAQEXLT+3NdL1fx+4fqZM2eUl5d33nHXrl2rO++8U3/961+VkJBw0fMZP368CgoK7K8DBw5ctB4AAFRfTg1RHh4e6ty5s9LS0uxjNptNaWlpioiIKHOfiIgIh3pJWrVqlb0+NDRUAQEBDjWFhYXKyMiw10RERCg/P1+ZmZn2mtWrV8tmsyk8PNw+tmbNGkVHR+svf/mLw2/uXYinp6d8fHwcXgAAoGZy6pooSRo9erQGDx6sLl26qGvXrpoxY4aKioo0ZMgQSVJCQoKaNGmipKQkSdKoUaPUs2dPvf7664qOjtaSJUu0adMmzZ07V5Lk4uKiJ598Ui+++KJatWql0NBQTZo0SUFBQYqNjZUktWvXTn369NHw4cOVnJys06dPKzExUQMHDrT/Zt5XX32l/v37a9SoURowYIB9rZSHhweLywEAgPNDVFxcnA4dOqTJkycrJydHYWFhSk1NtS8M//nnn+Xq+p8bZt27d9fixYs1ceJETZgwQa1atdKyZcvUoUMHe83YsWNVVFSkESNGKD8/XzfddJNSU1Pl5eVlr1m0aJESExPVq1cvubq6asCAAZo5c6Z9+4IFC3TixAklJSXZA5wk9ezZU2vWrLmM7wgAAKgOnP6cqJqM50QBwNWJ50RdGVf1c6IAAACqK0IUAACABYQoAAAACwhRAAAAFhCiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAAAALCBEAQAAWECIAgAAsIAQBQAAYAEhCgAAwAJCFAAAgAWEKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAAC9yd3QAAhIxb4ewWrhr7Xol2dgtAjcGdKAAAAAsIUQAAABYQogAAACwgRAEAAFjAwvJqioW4VwaLcAEAF8KdKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAALCFEAAAAWEKIAAAAsqBIhas6cOQoJCZGXl5fCw8O1YcOGi9anpKSobdu28vLy0vXXX6+VK1c6bDfGaPLkyQoMDJS3t7ciIyO1e/duh5q8vDzFx8fLx8dHfn5+GjZsmI4fP+5Q88MPP+jmm2+Wl5eXgoODNW3atMo5YQAAUO05PUR98MEHGj16tKZMmaLNmzerY8eOioqK0sGDB8usX79+vQYNGqRhw4bpu+++U2xsrGJjY7Vt2zZ7zbRp0zRz5kwlJycrIyNDderUUVRUlE6dOmWviY+P1/bt27Vq1SotX75cX3/9tUaMGGHfXlhYqN69e6t58+bKzMzUq6++queee05z5869fG8GAACoNlyMMcaZDYSHh+vGG2/U7NmzJUk2m03BwcF6/PHHNW7cuPPq4+LiVFRUpOXLl9vHunXrprCwMCUnJ8sYo6CgID399NMaM2aMJKmgoED+/v6aP3++Bg4cqKysLLVv314bN25Uly5dJEmpqanq16+ffvnlFwUFBemtt97Ss88+q5ycHHl4eEiSxo0bp2XLlmnnzp3lOrfCwkL5+vqqoKBAPj4+f+h9+r2QcSsqdT6Ubd8r0c5u4arA9XzlcE1fGVzTV8blup7L+/PbqXeiSkpKlJmZqcjISPuYq6urIiMjlZ6eXuY+6enpDvWSFBUVZa/Pzs5WTk6OQ42vr6/Cw8PtNenp6fLz87MHKEmKjIyUq6urMjIy7DW33HKLPUCdO86uXbt09OjRP3jmAACgunN35sEPHz6s0tJS+fv7O4z7+/tf8G5PTk5OmfU5OTn27efGLlbTuHFjh+3u7u5q0KCBQ01oaOh5c5zbVr9+/fN6Ky4uVnFxsf37goICSWcTbWWzFZ+o9Dlxvsvxzw7n43q+crimrwyu6Svjcl3P5+a91Id1Tg1RNU1SUpKmTp163nhwcLATukFl8J3h7A6AysU1jZrkcl/Px44dk6+v7wW3OzVENWzYUG5ubsrNzXUYz83NVUBAQJn7BAQEXLT+3Nfc3FwFBgY61ISFhdlrfr9w/cyZM8rLy3OYp6zj/Pcxfm/8+PEaPXq0/Xubzaa8vDxdc801cnFxKXOfq0VhYaGCg4N14MCBSl8fBjgD1zRqEq5nR8YYHTt2TEFBQRetc2qI8vDwUOfOnZWWlqbY2FhJZ4NHWlqaEhMTy9wnIiJCaWlpevLJJ+1jq1atUkREhCQpNDRUAQEBSktLs4emwsJCZWRk6JFHHrHPkZ+fr8zMTHXu3FmStHr1atlsNoWHh9trnn32WZ0+fVq1atWyH6dNmzZlfpQnSZ6envL09HQY8/Pzq/D7UpP5+PjwLyhqFK5p1CRcz/9xsTtQdsbJlixZYjw9Pc38+fPNjh07zIgRI4yfn5/JyckxxhjzwAMPmHHjxtnr161bZ9zd3c1rr71msrKyzJQpU0ytWrXM1q1b7TWvvPKK8fPzM5988on54YcfTExMjAkNDTUnT5601/Tp08f86U9/MhkZGebbb781rVq1MoMGDbJvz8/PN/7+/uaBBx4w27ZtM0uWLDG1a9c2f/vb367Au1LzFBQUGEmmoKDA2a0AlYJrGjUJ17M1Tg9Rxhgza9Ys06xZM+Ph4WG6du1q/vWvf9m39ezZ0wwePNih/sMPPzStW7c2Hh4e5rrrrjMrVqxw2G6z2cykSZOMv7+/8fT0NL169TK7du1yqDly5IgZNGiQqVu3rvHx8TFDhgwxx44dc6j5/vvvzU033WQ8PT1NkyZNzCuvvFK5J34V4V9Q1DRc06hJuJ6tcfpzonB1KC4uVlJSksaPH3/eR55AdcQ1jZqE69kaQhQAAIAFTv+zLwAAANURIQoAAMACQhQAAIAFhCgAAAALCFE1TFJSkm688UbVq1dPjRs3VmxsrHbt2uVQc+rUKT322GO65pprVLduXQ0YMMDh6ezff/+9Bg0apODgYHl7e6tdu3Z64403LnjMdevWyd3d3f5w04sxxmjy5MkKDAyUt7e3IiMjtXv37jJri4uLFRYWJhcXF23ZsuWSc69Zs0adOnWSp6enrr32Ws2fP99he2lpqSZNmqTQ0FB5e3urZcuWeuGFFy75t5FQtVS1a3zOnDkKCQmRl5eXwsPDtWHDBoftI0eOVMuWLeXt7a1GjRopJibG4W+DlreX4uJiPfvss2revLk8PT0VEhKid999t7xvG6qwq/WanjNnjtq1aydvb2+1adNG7733XnnfsqrDmc9XQOWLiooy8+bNM9u2bTNbtmwx/fr1M82aNTPHjx+31zz88MMmODjYpKWlmU2bNplu3bqZ7t2727e/88475oknnjBr1qwxe/bsMQsXLjTe3t5m1qxZ5x3v6NGjpkWLFqZ3796mY8eOl+zvlVdeMb6+vmbZsmXm+++/N3fdddd5D0I954knnjB9+/Y1ksx333130Xn37t1rateubUaPHm127NhhZs2aZdzc3Exqaqq95qWXXjLXXHONWb58ucnOzjYpKSmmbt265o033rhk36g6qtI1vmTJEuPh4WHeffdds337djN8+HDj5+dncnNz7TV/+9vfzNq1a012drbJzMw0d955pwkODjZnzpypUC933XWXCQ8PN6tWrTLZ2dlm/fr15ttvv62MtxROdjVe02+++aapV6+eWbJkidmzZ4/5xz/+YerWrWs+/fTTynpbrwhCVA138OBBI8msXbvWGHP2Sey1atUyKSkp9pqsrCwjyaSnp19wnkcffdTcdttt543HxcWZiRMnmilTplwyRNlsNhMQEGBeffVV+1h+fr7x9PQ0//jHPxxqV65cadq2bWu2b99erhA1duxYc911153XW1RUlP376OhoM3ToUIeae+65x8THx190blRtzrzGu3btah577DH796WlpSYoKMgkJSVd8Djff/+9kWR++umncvfy+eefG19fX3PkyJEL7oOa42q4piMiIsyYMWMcakaPHm169OhxwTmqIj7Oq+EKCgokSQ0aNJAkZWZm6vTp04qMjLTXtG3bVs2aNVN6evpF5zk3xznz5s3T3r17NWXKlHL1kp2drZycHIdj+/r6Kjw83OHYubm5Gj58uBYuXKjatWuXa+709HSHeSUpKirKYd7u3bsrLS1NP/74o6Szt5y//fZb9e3bt1zHQNXkrGu8pKREmZmZDsdxdXVVZGTkBY9TVFSkefPmKTQ0VMHBweXu5dNPP1WXLl00bdo0NWnSRK1bt9aYMWN08uTJC86B6utquKaLi4vl5eXlUOPt7a0NGzbo9OnTF5ynqnHqHyDG5WWz2fTkk0+qR48e6tChgyQpJydHHh4e5/1hZH9/f+Xk5JQ5z/r16/XBBx9oxYoV9rHdu3dr3Lhx+uabb+TuXr7L6Nz8/v7+Fzy2MUYPPvigHn74YXXp0kX79u0r99xlzVtYWKiTJ0/K29tb48aNU2Fhodq2bSs3NzeVlpbqpZdeUnx8fLmOgarHmdf44cOHVVpaWuZ199/rQyTpzTff1NixY1VUVKQ2bdpo1apV8vDwKHcve/fu1bfffisvLy99/PHHOnz4sB599FEdOXJE8+bNu/AbhGrnarmmo6Ki9Pbbbys2NladOnVSZmam3n77bZ0+fVqHDx9WYGDghd+kKoQ7UTXYY489pm3btmnJkiWW59i2bZtiYmI0ZcoU9e7dW9LZBdp//vOfNXXqVLVu3brM/RYtWqS6devaX9988025jjdr1iwdO3ZM48ePv2DNf8/78MMPl/tcPvzwQy1atEiLFy/W5s2btWDBAr322mtasGBBuedA1eLMa7wi4uPj9d1332nt2rVq3bq17rvvPp06dapcvUhnf7C6uLho0aJF6tq1q/r166fp06drwYIF3I2qYa6Wa3rSpEnq27evunXrplq1aikmJkaDBw+WdPbuV7Xh7M8TcXk89thjpmnTpmbv3r0O42lpaUaSOXr0qMN4s2bNzPTp0x3Gtm/fbho3bmwmTJjgMH706FEjybi5udlfLi4u9rG0tDRTWFhodu/ebX+dOHHC7Nmzp8z1Tbfccot54oknjDHGxMTEGFdXV4e5z82bkJBgjDEO855b7HjzzTebUaNGOcz77rvvGh8fH/v3TZs2NbNnz3aoeeGFF0ybNm0u/YaiynH2NV5cXGzc3NzMxx9/7LBvQkKCueuuuy7Yd3Fxsaldu7ZZvHhxuXo5N2fLli0dxnbs2GEkmR9//PGCx0L1cjVd0+eUlJSYAwcOmDNnztgXm5eWll6wvqohRNUwNpvNPPbYYyYoKKjM/7ieW6C4dOlS+9jOnTvPW6C4bds207hxY/PMM8+cN0dpaanZunWrw+uRRx4xbdq0MVu3bnX4jZLf9xYQEGBee+01+1hBQYHDwvL9+/c7zPvFF18YSWbp0qXmwIEDFzzvsWPHmg4dOjiMDRo0yGFheYMGDcybb77pUPPyyy+bVq1aXXBeVD1V6Rrv2rWrSUxMdNivSZMmF12Ee+rUKePt7W3mzZtXrl6MOfvbUN7e3ubYsWP2sWXLlhlXV1dz4sSJCx4L1cPVeE2X5ZZbbjGDBg0qd31VQIiqYR555BHj6+tr1qxZY3777Tf767//Q/vwww+bZs2amdWrV5tNmzaZiIgIExERYd++detW06hRI3P//fc7zHHw4MELHrc8v51nzNlHHPj5+ZlPPvnE/PDDDyYmJuaCjzgwxpjs7OwKPeLgmWeeMVlZWWbOnDnnPeJg8ODBpkmTJvZHHHz00UemYcOGZuzYsZfsG1VHVbrGlyxZYjw9Pc38+fPNjh07zIgRI4yfn5/JyckxxhizZ88e8/LLL5tNmzaZ/fv3m3Xr1pk777zTNGjQwH4XtTy9HDt2zDRt2tTce++9Zvv27Wbt2rWmVatW5qGHHqqMtxROdjVe07t27TILFy40P/74o8nIyDBxcXGmQYMGJjs7uxLe0SuHEFXDSCrz9d//h3Dy5Enz6KOPmvr165vatWubu+++2/z222/27VOmTClzjubNm1/wuOUNUTabzUyaNMn4+/sbT09P06tXL7Nr164L1pc3RBljzFdffWXCwsKMh4eHadGihcM5G2NMYWGhGTVqlGnWrJnx8vIyLVq0MM8++6wpLi6+5NyoOqraNT5r1izTrFkz4+HhYbp27Wr+9a9/2bf9+9//Nn379jWNGzc2tWrVMk2bNjV//vOfzc6dOyvcS1ZWlomMjDTe3t6madOmZvTo0dyFqiGuxmt6x44dJiwszHh7exsfHx8TExPjMEd14WIMj2sGAACoqGq0BB4AAKDqIEQBAABYQIgCAACwgBAFAABgASEKAADAAkIUAACABYQoAAAACwhRAAAAFhCiAEDSgw8+KBcXF7m4uKhWrVry9/fXHXfcoXfffVc2m83Z7QGogghRAPB/+vTpo99++0379u3T559/rttuu02jRo1S//79debMGWe3B6CKIUQBwP/x9PRUQECAmjRpok6dOmnChAn65JNP9Pnnn2v+/PmSpOnTp+v6669XnTp1FBwcrEcffVTHjx+XJBUVFcnHx0dLly51mHfZsmWqU6eOjh07dqVPCcBlRIgCgIu4/fbb1bFjR3300UeSJFdXV82cOVPbt2/XggULtHr1ao0dO1aSVKdOHQ0cOFDz5s1zmGPevHm69957Va9evSveP4DLhz9ADAA6uyYqPz9fy5YtO2/bwIED9cMPP2jHjh3nbVu6dKkefvhhHT58WJK0YcMGde/eXQcOHFBgYKAOHjyoJk2a6Msvv1TPnj0v92kAuIK4EwUAl2CMkYuLiyTpyy+/VK9evdSkSRPVq1dPDzzwgI4cOaITJ05Ikrp27arrrrtOCxYskCS9//77at68uW655Ran9Q/g8iBEAcAlZGVlKTQ0VPv27VP//v11ww036J///KcyMzM1Z84cSVJJSYm9/qGHHrKvoZo3b56GDBliD2EAag5CFABcxOrVq7V161YNGDBAmZmZstlsev3119WtWze1bt1av/7663n73H///dq/f79mzpypHTt2aPDgwU7oHMDl5u7sBgCgqiguLlZOTo5KS0uVm5ur1NRUJSUlqX///kpISNC2bdt0+vRpzZo1S3feeafWrVun5OTk8+apX7++7rnnHj3zzDPq3bu3mjZt6oSzAXC5cScKAP5PamqqAgMDFRISoj59+uirr77SzJkz9cknn8jNzU0dO3bU9OnT9Ze//EUdOnTQokWLlJSUVOZcw4YNU0lJiYYOHXqFzwLAlcJv5wHAZbBw4UI99dRT+vXXX+Xh4eHsdgBcBnycBwCV6MSJE/rtt9/0yiuvaOTIkQQooAbj4zwAqETTpk1T27ZtFRAQoPHjxzu7HQCXER/nAQAAWMCdKAAAAAsIUQAAABYQogAAACwgRAEAAFhAiAIAALCAEAUAAGABIQoAAMACQhQAAIAFhCgAAAAL/j+tm/sa9GvIcgAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAHWCAYAAAD6oMSKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3hUlEQVR4nO3deVyVZf7/8fcBBNzAFVAjUEvNTFFcQjM1MTQ1dWpCcwQtTSvKpDKp3FqkTQcrCjWVTB01p7TEaFwzjcmvOJiVS+6OCu4CLiCc+/dHP09zAvWgwIGb1/PxOI+8r3Nd5/7c+OjyzX3f130shmEYAgAAQLnn4uwCAAAAUDwIdgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCQIdgBwAxITE2WxWLRlyxan7L9r167q2rWrU/YNoOwi2AEok/bu3auRI0eqUaNG8vT0lJeXlzp16qTp06fr4sWLxb6/CxcuaNKkSVq/fn2xfzYAlBY3ZxcAAH+WlJSkv/71r/Lw8FBERIRatGih3Nxcbdy4US+++KJ++eUXzZw5s1j3eeHCBU2ePFmSOBMGoNwi2AEoU/bv36+BAwcqICBAa9euVb169WzvPf3009qzZ4+SkpKcWCEAlF1cigVQprzzzjvKzs7W7Nmz7ULdFbfddptGjx5t287Ly9Prr7+uxo0by8PDQ4GBgXr55ZeVk5NjN27Lli0KCwtTnTp1VLlyZTVs2FCPPfaYJOnAgQOqW7euJGny5MmyWCyyWCyaNGnSdeu9cOGCRo4cqdq1a8vLy0sRERE6c+aM7f3IyEjVqVNHly9fLjD2/vvvV9OmTa+7j5kzZ6px48aqXLmy2rdvr++//75An9zcXE2YMEHBwcHy9vZW1apV1blzZ61bt87WxzAMBQYGql+/fgXGX7p0Sd7e3ho5cuR16wFQdhHsAJQpX3/9tRo1aqSOHTs61H/48OGaMGGC2rRpo7///e/q0qWLYmNjNXDgQFuf48eP6/7779eBAwc0btw4ffDBBxo8eLD+/e9/S5Lq1q2rjz/+WJI0YMAAffbZZ/rss8/0l7/85br7j4qK0o4dOzRp0iRFRERowYIF6t+/vwzDkCQNGTJEp06d0rfffms3Lj09XWvXrtXf/va3a37+7NmzNXLkSPn5+emdd95Rp06d9OCDD+rw4cN2/TIzM/XJJ5+oa9euevvttzVp0iSdOHFCYWFhSktLkyRZLBb97W9/0zfffKPTp0/bjf/666+VmZl53XoAlHEGAJQR586dMyQZ/fr1c6h/WlqaIckYPny4XfsLL7xgSDLWrl1rGIZhfPnll4Yk4//+7/+u+lknTpwwJBkTJ050aN9z5841JBnBwcFGbm6urf2dd94xJBnLly83DMMw8vPzjVtuucUIDw+3Gz9t2jTDYrEY+/btu+o+cnNzDR8fHyMoKMjIycmxtc+cOdOQZHTp0sXWlpeXZ9fHMAzjzJkzhq+vr/HYY4/Z2nbt2mVIMj7++GO7vg8++KARGBhoWK1Wh44fQNnEGTsAZUZmZqYkqXr16g71X7lypSQpOjrarv3555+XJNu9eDVq1JAkrVixotBLojfjiSeeUKVKlWzbTz75pNzc3Gy1ubi4aPDgwfrqq6+UlZVl67dgwQJ17NhRDRs2vOpnb9myRcePH9eoUaPk7u5uax86dKi8vb3t+rq6utr6WK1WnT59Wnl5eWrbtq22bt1q69ekSRN16NBBCxYssLWdPn1a33zzjQYPHiyLxXKDPwkAZUGFDnYbNmxQ3759Vb9+fVksFi1btqxE9zdp0iTbvTtXXs2aNSvRfQLliZeXlyTZBaBrOXjwoFxcXHTbbbfZtfv5+alGjRo6ePCgJKlLly566KGHNHnyZNWpU0f9+vXT3LlzC9yHdyNuv/12u+1q1aqpXr16OnDggK0tIiJCFy9e1JdffilJ2rVrl1JTUzVkyJDrHl9h+6hUqZIaNWpUoP+nn36qli1bytPTU7Vr11bdunWVlJSkc+fO2fWLiIjQpk2bbJ//+eef6/Lly9etB0DZV6GD3fnz59WqVSvFx8eX2j7vvPNOHTt2zPbauHFjqe0bKOu8vLxUv359/fzzz0Uad72zTBaLRUuXLlVKSoqioqJ05MgRPfbYYwoODlZ2dvbNlOyQ5s2bKzg4WPPnz5ckzZ8/X+7u7nrkkUeKbR/z58/X0KFD1bhxY82ePVvJyclatWqV7rvvPlmtVru+AwcOVKVKlWxn7ebPn6+2bds6tJADQNlWoYNdr1699MYbb2jAgAGFvp+Tk6MXXnhBDRo0UNWqVdWhQ4ebfnipm5ub/Pz8bK86derc1OcBZtOnTx/t3btXKSkp1+0bEBAgq9Wq3377za49IyNDZ8+eVUBAgF373XffrTfffFNbtmzRggUL9Msvv2jRokWSrh8Or+bP+87OztaxY8cUGBho1x4REaG1a9fq2LFjWrhwoXr37q2aNWte9/gK28fly5e1f/9+u7alS5eqUaNG+uKLLzRkyBCFhYUpNDRUly5dKvC5tWrVUu/evbVgwQIdPHhQmzZt4mwdYBIVOthdT1RUlFJSUrRo0SL99NNP+utf/6qePXsWmGSL4rffflP9+vXVqFEjDR48WIcOHSrGioHyb+zYsapataqGDx+ujIyMAu/v3btX06dPlyQ98MADkqS4uDi7PtOmTZMk9e7dW5J05swZ2yrVK4KCgiTJdjm2SpUqkqSzZ88Wqd6ZM2fa3bf38ccfKy8vT7169bLrN2jQIFksFo0ePVr79u1zaPVp27ZtVbduXSUkJCg3N9fWnpiYWKBOV1dXSbI7zh9//PGqAXnIkCH69ddf9eKLL8rV1dVuFTGAcszZqzfKCknGl19+ads+ePCg4erqahw5csSuX/fu3Y2YmJgb2sfKlSuNJUuWGNu2bTOSk5ONkJAQ49ZbbzUyMzNvpnTAdJYvX254enoaNWvWNEaPHm3MmjXLiI+PNwYPHmy4u7sbTzzxhK1vZGSkIcl45JFHjPj4eNt2//79bX3+/ve/G7fffrsxduxYY8aMGcZ7771nNG3a1PDy8rJbldq8eXPDz8/PiI+PN/7xj38Y27dvv2qNV1bF3nXXXUbnzp2NDz74wIiKijJcXFyMe+65p9DVpX369DEkGTVq1DAuXbrk0M9ixowZhiSjU6dOxvvvv2+MGTPGqFGjhtGoUSO7VbFz5swxJBkPPvigMWPGDGPcuHFGjRo1jDvvvNMICAgo8Lk5OTlG7dq1DUlGr169HKoFQNlHsPv//hzsVqxYYUgyqlatavdyc3MzHnnkEcMwDGPHjh2GpGu+Xnrppavu88yZM4aXl5fxySeflPThAeXO7t27jREjRhiBgYGGu7u7Ub16daNTp07GBx98YBeKLl++bEyePNlo2LChUalSJcPf39+IiYmx67N161Zj0KBBxq233mp4eHgYPj4+Rp8+fYwtW7bY7fOHH34wgoODDXd39+s++uRKsPvuu++MJ554wqhZs6ZRrVo1Y/DgwcapU6cKHbNkyRJDkl0wdcRHH31kNGzY0PDw8DDatm1rbNiwwejSpYtdsLNarcaUKVOMgIAAw8PDw2jdurWxYsUKIzIystBgZxiG8dRTTxmSjIULFxapHgBll8Uw/nR9ooKyWCz68ssv1b9/f0nS4sWLNXjwYP3yyy+2SxxXVKtWTX5+fsrNzdW+ffuu+blXVqZdTbt27RQaGqrY2NibPgYAZdvy5cvVv39/bdiwQZ07d3Z2ORozZoxmz56t9PR026VoAOUb3xV7Fa1bt1Z+fr6OHz9+1QnY3d39ph5Xkp2drb1793LTMlBBzJo1S40aNdI999zj7FJ06dIlzZ8/Xw899BChDjCRCh3ssrOztWfPHtv2/v37lZaWplq1aqlJkyYaPHiwIiIiNHXqVLVu3VonTpzQmjVr1LJlS9tN2UXxwgsvqG/fvgoICNDRo0c1ceJEubq6atCgQcV5WADKmCsLsJKSkjR9+nSnPgT4+PHjWr16tZYuXapTp07Zfe8ugPKvQl+KXb9+vbp161agPTIyUomJibp8+bLeeOMNzZs3T0eOHFGdOnV09913a/LkybrrrruKvL+BAwdqw4YNOnXqlOrWrat77rlHb775pho3blwchwOgjLJYLKpWrZrCw8OVkJAgNzfn/U59Zd7z8fHR+PHjFRUV5bRaABS/Ch3sAAAAzITn2AEAAJgEwQ4AAMAkKtziCavVqqNHj6p69epOvYEZAADAEYZhKCsrS/Xr15eLy7XPyVW4YHf06FH5+/s7uwwAAIAiOXz4sG655ZZr9qlwwa569eqSfv/heHl5ObkaAACAa8vMzJS/v78tw1xLhQt2Vy6/enl5EewAAEC54cgtZCyeAAAAMAmCHQAAgEkQ7AAAAEyCYAcAAGASBDsAAACTINgBAACYBMEOAADAJAh2AAAAJkGwAwAAMAmCHQAAgEkQ7AAAAEyCYAcAAGASBDsAAACTINgBAACYhJuzCzCzwHFJzi4BuKYDb/V2dgnANTGPojwoS3MpZ+wAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCQIdgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCScGuw2bNigvn37qn79+rJYLFq2bJnDYzdt2iQ3NzcFBQWVWH0AAADliVOD3fnz59WqVSvFx8cXadzZs2cVERGh7t27l1BlAAAA5Y+bM3feq1cv9erVq8jjRo0apUcffVSurq5FOssHAABgZuXuHru5c+dq3759mjhxorNLAQAAKFOcesauqH777TeNGzdO33//vdzcHCs9JydHOTk5tu3MzMySKg8AAMCpys0Zu/z8fD366KOaPHmymjRp4vC42NhYeXt7217+/v4lWCUAAIDzlJtgl5WVpS1btigqKkpubm5yc3PTa6+9pm3btsnNzU1r164tdFxMTIzOnTtnex0+fLiUKwcAACgd5eZSrJeXl7Zv327X9tFHH2nt2rVaunSpGjZsWOg4Dw8PeXh4lEaJAAAATuXUYJedna09e/bYtvfv36+0tDTVqlVLt956q2JiYnTkyBHNmzdPLi4uatGihd14Hx8feXp6FmgHAACoiJwa7LZs2aJu3brZtqOjoyVJkZGRSkxM1LFjx3To0CFnlQcAAFCuODXYde3aVYZhXPX9xMTEa46fNGmSJk2aVLxFAQAAlFPlZvEEAAAAro1gBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJNwarDbsGGD+vbtq/r168tisWjZsmXX7P/FF1+oR48eqlu3rry8vBQSEqJvv/22dIoFAAAo45wa7M6fP69WrVopPj7eof4bNmxQjx49tHLlSqWmpqpbt27q27ev/vOf/5RwpQAAAGWfmzN33qtXL/Xq1cvh/nFxcXbbU6ZM0fLly/X111+rdevWxVwdAABA+VKu77GzWq3KyspSrVq1nF0KAACA0zn1jN3Neu+995Sdna1HHnnkqn1ycnKUk5Nj287MzCyN0gAAAEpduT1jt3DhQk2ePFlLliyRj4/PVfvFxsbK29vb9vL39y/FKgEAAEpPuQx2ixYt0vDhw7VkyRKFhoZes29MTIzOnTtnex0+fLiUqgQAAChd5e5S7D/+8Q899thjWrRokXr37n3d/h4eHvLw8CiFygAAAJzLqcEuOztbe/bssW3v379faWlpqlWrlm699VbFxMToyJEjmjdvnqTfL79GRkZq+vTp6tChg9LT0yVJlStXlre3t1OOAQAAoKxw6qXYLVu2qHXr1rZHlURHR6t169aaMGGCJOnYsWM6dOiQrf/MmTOVl5enp59+WvXq1bO9Ro8e7ZT6AQAAyhKnnrHr2rWrDMO46vuJiYl22+vXry/ZggAAAMqxcrl4AgAAAAUR7AAAAEyCYAcAAGASBDsAAACTINgBAACYBMEOAADAJAh2AAAAJkGwAwAAMAmCHQAAgEkQ7AAAAEyCYAcAAGASBDsAAACTINgBAACYBMEOAADAJAh2AAAAJkGwAwAAMAmCHQAAgEkQ7AAAAEyCYAcAAGASBDsAAACTINgBAACYBMEOAADAJAh2AAAAJkGwAwAAMAmCHQAAgEkQ7AAAAEyCYAcAAGASBDsAAACTINgBAACYBMEOAADAJAh2AAAAJkGwAwAAMAmCHQAAgEkQ7AAAAEyCYAcAAGASTg12GzZsUN++fVW/fn1ZLBYtW7bsumPWr1+vNm3ayMPDQ7fddpsSExNLvE4AAIDywKnB7vz582rVqpXi4+Md6r9//3717t1b3bp1U1pamp577jkNHz5c3377bQlXCgAAUPa5OXPnvXr1Uq9evRzun5CQoIYNG2rq1KmSpDvuuEMbN27U3//+d4WFhZVUmQAAAOVCubrHLiUlRaGhoXZtYWFhSklJcVJFAAAAZYdTz9gVVXp6unx9fe3afH19lZmZqYsXL6py5coFxuTk5CgnJ8e2nZmZWeJ1AgAAOEO5OmN3I2JjY+Xt7W17+fv7O7skAACAElGugp2fn58yMjLs2jIyMuTl5VXo2TpJiomJ0blz52yvw4cPl0apAAAApa5cXYoNCQnRypUr7dpWrVqlkJCQq47x8PCQh4dHSZcGAADgdE49Y5edna20tDSlpaVJ+v1xJmlpaTp06JCk38+2RURE2PqPGjVK+/bt09ixY7Vz50599NFHWrJkicaMGeOM8gEAAMoUpwa7LVu2qHXr1mrdurUkKTo6Wq1bt9aECRMkSceOHbOFPElq2LChkpKStGrVKrVq1UpTp07VJ598wqNOAAAA5ORLsV27dpVhGFd9v7Bvlejatav+85//lGBVAAAA5VO5WjwBAACAqyPYAQAAmATBDgAAwCQIdgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCQIdgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCQIdgAAACbh5kinWrVqFelDLRaLtm7dqoCAgBsqCgAAAEXnULA7e/as4uLi5O3tfd2+hmHoqaeeUn5+/k0XBwAAAMc5FOwkaeDAgfLx8XGo7zPPPHPDBQEAAODGOBTsrFZrkT40KyvrhooBAADAjWPxBAAAgEncdLDbsWOH5s6dq7S0tGIoBwAAADfK4XvsJOm1115T5cqV9eKLL0qS1q1bp549e6p69eo6d+6cEhMTNXjw4BIpFAAAANdWpDN2S5cuVfPmzW3bb775pp599lmdPHlSH374oaZMmVLsBQIAAMAxDp2xmzdvngzD0IEDB5SWlqZTp07JMAxt2rRJnTt31rx582S1WrVv3z7NmzdPkhQREVGihQMAAMCeQ8HuyoOG3d3d5evrq4CAAKWlpcnLy0vdunWTYRjKycmRxWJRYGCgDMMo0aIBAABQkEPBrkuXLpKkNm3aaMWKFXrppZeUnJysBx54QPfee68kafv27fL397dtAwAAoHQV6R67d999V2lpaerUqZMOHjyo1157zfZeYmKievbsWewFAgAAwDFFWhXbqlUrHThwQKdOnVLt2rXt3nvhhRfk5eVVrMUBAADAcUUKdlf8OdRJUr169W66GAAAANw4hy7Fvv/++7p06ZLDH5qQkMDXigEAAJQyh4LdmDFjihTUxo4dqxMnTjjUNz4+XoGBgfL09FSHDh20efPma/aPi4tT06ZNVblyZfn7+2vMmDFFCp0AAABm5dClWMMw1L17d7m5OXbl9uLFiw71W7x4saKjo5WQkKAOHTooLi5OYWFh2rVrl3x8fAr0X7hwocaNG6c5c+aoY8eO2r17t4YOHSqLxaJp06Y5tE8AAACzciipTZw4sUgf2q9fP9WqVeu6/aZNm6YRI0Zo2LBhkn6/hJuUlKQ5c+Zo3LhxBfr/8MMP6tSpkx599FFJUmBgoAYNGqQff/yxSPUBAACYUYkEO0fk5uYqNTVVMTExtjYXFxeFhoYqJSWl0DEdO3bU/PnztXnzZrVv31779u3TypUrNWTIkGKvDwAAoLy5oVWxxeHkyZPKz8+Xr6+vXbuvr6927txZ6JhHH31UJ0+e1D333CPDMJSXl6dRo0bp5Zdfvup+cnJylJOTY9vOzMwsngMAAAAoY4r0gGJnW79+vaZMmaKPPvpIW7du1RdffKGkpCS9/vrrVx0TGxsrb29v28vf378UKwYAACg9TjtjV6dOHbm6uiojI8OuPSMjQ35+foWOGT9+vIYMGaLhw4dLku666y6dP39eTzzxhF555RW5uBTMqTExMYqOjrZtZ2ZmEu4AAIApOe2Mnbu7u4KDg7VmzRpbm9Vq1Zo1axQSElLomAsXLhQIb66urpJ+X7lbGA8PD3l5edm9AAAAzKjIwe61117ThQsXCrRfvHjR7rtjHREdHa1Zs2bp008/1Y4dO/Tkk0/q/PnztlWyERERdosr+vbtq48//liLFi3S/v37tWrVKo0fP159+/a1BTwAAICKqsiXYidPnqxRo0apSpUqdu0XLlzQ5MmTNWHCBIc/Kzw8XCdOnNCECROUnp6uoKAgJScn2xZUHDp0yO4M3auvviqLxaJXX31VR44cUd26ddW3b1+9+eabRT0MAAAA0ylysDMMQxaLpUD7tm3bHHp23Z9FRUUpKiqq0PfWr19vt+3m5qaJEyeWyONXAAAAyjuHg13NmjVlsVhksVjUpEkTu3CXn5+v7OxsjRo1qkSKBAAAwPU5HOzi4uJkGIYee+wxTZ48Wd7e3rb33N3dFRgYeNVFDwAAACh5Dge7yMhISVLDhg3VqVMnh783FgAAAKWjyKtiq1evrh07dti2ly9frv79++vll19Wbm5usRYHAAAAxxU52I0cOVK7d++WJO3bt0/h4eGqUqWKPv/8c40dO7bYCwQAAIBjihzsdu/eraCgIEnS559/ri5dumjhwoVKTEzUP//5z+KuDwAAAA4qcrAzDENWq1WStHr1aj3wwAOSJH9/f508ebJ4qwMAAIDDihzs2rZtqzfeeEOfffaZvvvuO/Xu3VuStH//ftuDhQEAAFD6ihzs4uLitHXrVkVFRemVV17RbbfdJklaunSpOnbsWOwFAgAAwDFFfmZJy5YttX379gLt7777Lt/XCgAA4EQ3/DC61NRU22NPmjdvrjZt2hRbUQAAACi6Ige748ePKzw8XN99951q1KghSTp79qy6deumRYsWqW7dusVdIwAAABxQ5HvsnnnmGWVnZ+uXX37R6dOndfr0af3888/KzMzUs88+WxI1AgAAwAFFPmOXnJys1atX64477rC1NW/eXPHx8br//vuLtTgAAAA4rshn7KxWqypVqlSgvVKlSrbn2wEAAKD0FTnY3XfffRo9erSOHj1qazty5IjGjBmj7t27F2txAAAAcFyRg92HH36ozMxMBQYGqnHjxmrcuLEaNmyozMxMffDBByVRIwAAABxQ5Hvs/P39tXXrVq1evVo7d+6UJN1xxx0KDQ0t9uIAAADguBt6jp3FYlGPHj3Uo0eP4q4HAAAAN8jhS7Fr165V8+bNlZmZWeC9c+fO6c4779T3339frMUBAADAcQ4Hu7i4OI0YMUJeXl4F3vP29tbIkSM1bdq0Yi0OAAAAjnM42G3btk09e/a86vv333+/UlNTi6UoAAAAFJ3DwS4jI6PQ59dd4ebmphMnThRLUQAAACg6h4NdgwYN9PPPP1/1/Z9++kn16tUrlqIAAABQdA4HuwceeEDjx4/XpUuXCrx38eJFTZw4UX369CnW4gAAAOA4hx938uqrr+qLL75QkyZNFBUVpaZNm0qSdu7cqfj4eOXn5+uVV14psUIBAABwbQ4HO19fX/3www968sknFRMTI8MwJP3+TLuwsDDFx8fL19e3xAoFAADAtRXpAcUBAQFauXKlzpw5oz179sgwDN1+++2qWbNmSdUHAAAAB93QN0/UrFlT7dq1K+5aAAAAcBMcXjwBAACAso1gBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAm4fRgFx8fr8DAQHl6eqpDhw7avHnzNfufPXtWTz/9tOrVqycPDw81adJEK1euLKVqAQAAyq4beo5dcVm8eLGio6OVkJCgDh06KC4uTmFhYdq1a5d8fHwK9M/NzVWPHj3k4+OjpUuXqkGDBjp48KBq1KhR+sUDAACUMU4NdtOmTdOIESM0bNgwSVJCQoKSkpI0Z84cjRs3rkD/OXPm6PTp0/rhhx9UqVIlSVJgYGBplgwAAFBmOe1SbG5urlJTUxUaGvpHMS4uCg0NVUpKSqFjvvrqK4WEhOjpp5+Wr6+vWrRooSlTpig/P7+0ygYAACiznHbG7uTJk8rPz5evr69du6+vr3bu3FnomH379mnt2rUaPHiwVq5cqT179uipp57S5cuXNXHixELH5OTkKCcnx7admZlZfAcBAABQhjh98URRWK1W+fj4aObMmQoODlZ4eLheeeUVJSQkXHVMbGysvL29bS9/f/9SrBgAAKD0OC3Y1alTR66ursrIyLBrz8jIkJ+fX6Fj6tWrpyZNmsjV1dXWdscddyg9PV25ubmFjomJidG5c+dsr8OHDxffQQAAAJQhTgt27u7uCg4O1po1a2xtVqtVa9asUUhISKFjOnXqpD179shqtdradu/erXr16snd3b3QMR4eHvLy8rJ7AQAAmJFTL8VGR0dr1qxZ+vTTT7Vjxw49+eSTOn/+vG2VbEREhGJiYmz9n3zySZ0+fVqjR4/W7t27lZSUpClTpujpp5921iEAAACUGU593El4eLhOnDihCRMmKD09XUFBQUpOTrYtqDh06JBcXP7Inv7+/vr22281ZswYtWzZUg0aNNDo0aP10ksvOesQAAAAygynBjtJioqKUlRUVKHvrV+/vkBbSEiI/v3vf5dwVQAAAOVPuVoVCwAAgKsj2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkykSwi4+PV2BgoDw9PdWhQwdt3rzZoXGLFi2SxWJR//79S7ZAAACAcsDpwW7x4sWKjo7WxIkTtXXrVrVq1UphYWE6fvz4NccdOHBAL7zwgjp37lxKlQIAAJRtTg9206ZN04gRIzRs2DA1b95cCQkJqlKliubMmXPVMfn5+Ro8eLAmT56sRo0alWK1AAAAZZdTg11ubq5SU1MVGhpqa3NxcVFoaKhSUlKuOu61116Tj4+PHn/88dIoEwAAoFxwc+bOT548qfz8fPn6+tq1+/r6aufOnYWO2bhxo2bPnq20tDSH9pGTk6OcnBzbdmZm5g3XCwAAUJY5/VJsUWRlZWnIkCGaNWuW6tSp49CY2NhYeXt7217+/v4lXCUAAIBzOPWMXZ06deTq6qqMjAy79oyMDPn5+RXov3fvXh04cEB9+/a1tVmtVkmSm5ubdu3apcaNG9uNiYmJUXR0tG07MzOTcAcAAEzJqcHO3d1dwcHBWrNmje2RJVarVWvWrFFUVFSB/s2aNdP27dvt2l599VVlZWVp+vTphQY2Dw8PeXh4lEj9AAAAZYlTg50kRUdHKzIyUm3btlX79u0VFxen8+fPa9iwYZKkiIgINWjQQLGxsfL09FSLFi3sxteoUUOSCrQDAABUNE4PduHh4Tpx4oQmTJig9PR0BQUFKTk52bag4tChQ3JxKVe3AgIAADiF04OdJEVFRRV66VWS1q9ff82xiYmJxV8QAABAOcSpMAAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkykSwi4+PV2BgoDw9PdWhQwdt3rz5qn1nzZqlzp07q2bNmqpZs6ZCQ0Ov2R8AAKCicHqwW7x4saKjozVx4kRt3bpVrVq1UlhYmI4fP15o//Xr12vQoEFat26dUlJS5O/vr/vvv19Hjhwp5coBAADKFqcHu2nTpmnEiBEaNmyYmjdvroSEBFWpUkVz5swptP+CBQv01FNPKSgoSM2aNdMnn3wiq9WqNWvWlHLlAAAAZYtTg11ubq5SU1MVGhpqa3NxcVFoaKhSUlIc+owLFy7o8uXLqlWrVkmVCQAAUC64OXPnJ0+eVH5+vnx9fe3afX19tXPnToc+46WXXlL9+vXtwuH/ysnJUU5Ojm07MzPzxgsGAAAow5x+KfZmvPXWW1q0aJG+/PJLeXp6FtonNjZW3t7etpe/v38pVwkAAFA6nBrs6tSpI1dXV2VkZNi1Z2RkyM/P75pj33vvPb311lv617/+pZYtW161X0xMjM6dO2d7HT58uFhqBwAAKGucGuzc3d0VHBxst/DhykKIkJCQq45755139Prrrys5OVlt27a95j48PDzk5eVl9wIAADAjp95jJ0nR0dGKjIxU27Zt1b59e8XFxen8+fMaNmyYJCkiIkINGjRQbGysJOntt9/WhAkTtHDhQgUGBio9PV2SVK1aNVWrVs1pxwEAAOBsTg924eHhOnHihCZMmKD09HQFBQUpOTnZtqDi0KFDcnH548Tixx9/rNzcXD388MN2nzNx4kRNmjSpNEsHAAAoU5we7CQpKipKUVFRhb63fv16u+0DBw6UfEEAAADlULleFQsAAIA/EOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCQIdgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCQIdgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCQIdgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgAAwCQIdgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEmUi2MXHxyswMFCenp7q0KGDNm/efM3+n3/+uZo1ayZPT0/dddddWrlyZSlVCgAAUHY5PdgtXrxY0dHRmjhxorZu3apWrVopLCxMx48fL7T/Dz/8oEGDBunxxx/Xf/7zH/Xv31/9+/fXzz//XMqVAwAAlC1OD3bTpk3TiBEjNGzYMDVv3lwJCQmqUqWK5syZU2j/6dOnq2fPnnrxxRd1xx136PXXX1ebNm304YcflnLlAAAAZYtTg11ubq5SU1MVGhpqa3NxcVFoaKhSUlIKHZOSkmLXX5LCwsKu2h8AAKCicHPmzk+ePKn8/Hz5+vratfv6+mrnzp2FjklPTy+0f3p6eqH9c3JylJOTY9s+d+6cJCkzM/NmSneINedCie8DuBml8f8BcDOYR1EelPRceuXzDcO4bl+nBrvSEBsbq8mTJxdo9/f3d0I1QNniHefsCgCg/CutuTQrK0ve3t7X7OPUYFenTh25uroqIyPDrj0jI0N+fn6FjvHz8ytS/5iYGEVHR9u2rVarTp8+rdq1a8tisdzkEaC0ZGZmyt/fX4cPH5aXl5ezywGAcom5tHwyDENZWVmqX7/+dfs6Ndi5u7srODhYa9asUf/+/SX9HrzWrFmjqKioQseEhIRozZo1eu6552xtq1atUkhISKH9PTw85OHhYddWo0aN4igfTuDl5cVkBAA3ibm0/LnembornH4pNjo6WpGRkWrbtq3at2+vuLg4nT9/XsOGDZMkRUREqEGDBoqNjZUkjR49Wl26dNHUqVPVu3dvLVq0SFu2bNHMmTOdeRgAAABO5/RgFx4erhMnTmjChAlKT09XUFCQkpOTbQskDh06JBeXPxbvduzYUQsXLtSrr76ql19+WbfffruWLVumFi1aOOsQAAAAygSL4cgSC8DJcnJyFBsbq5iYmAKX1gEAjmEuNT+CHQAAgEk4/ZsnAAAAUDwIdgAAACZBsAMAADAJgh3w/3G7KQDcnLy8PGeXUOER7ABJ8fHxevXVV3Xs2DFnlwIA5VJiYqJCQkKUnJzs7FIqNKc/xw5wpqNHj2rIkCGyWCyaMmWKatWq5eySAKBcWb9+vd5++21duHBBGRkZqlOnjrNLqtA4Y4cKbfv27WrXrp1Wr16t9u3bO7scACgXDMOQYRhauHChYmJiNGzYMH333Xdq166dvvrqK1sflD7O2KFCW716tS5evChJGj9+vA4cOKB27dopNDRUzZs3l9VqtfvmEwCAZLFYJEmdOnXSo48+amtv06aNzpw5o7y8PLm5ETGcgX+xUGHMmzdPTz75pD766CMdP35cktSqVSvl5+crMjJSWVlZGjBggLZt26bHH39ckgh1APA/rsyjH374oU6dOqWAgABJfyyauHjxovbv3y83NzdZrVZnllph8a8WTG/fvn0KDQ3V/Pnz1bNnT82aNUsffvih8vPz5efnp+PHj+vgwYOKi4vTX/7yF8XGxiovL08//vijs0sHgDLhz/Po7Nmz9f777yszM1PSH78EDxgwQD/99JPOnDnDL8ZOwk8dpmYYhrZs2aK+ffvqX//6l/r166cJEybo66+/lqurq7p3766goCDl5eVp06ZNkqQDBw4oICBALVu2dHL1AOB8hc2j48eP14oVK+Tl5SXpj2CXn5+vO++8U+np6c4suULjAjhMzWKxKCQkRFWrVpX0+wR19913y8vLS6dPn1atWrU0aNAgeXh4KCoqSv369dM///lPDRkyRJUrV5ZhGLZ7SQCgIipsHg0JCVG1atV07tw5eXt72/refvvt+vbbb/Xaa685q9wKj2AH0/nzggd/f3/bny0Wi5KTk1WrVi3bo01uu+02jR07VsHBwfr111+1bNkyNW7c2NYfACoaR+fR/w11klSzZk29/vrr8vPz4xdjJ7EYrEeGCfz888/66aef7FZnXXFlcrny34cfflgDBgzQ4MGDtWfPHuXm5qp58+ZOqBoAyo6bmUfz8vLUrFkzJ1SNP+OMHUyhT58+8vT0lL+/vzp37iyr1SrDMOTq6mr7jfHKf6tUqaJGjRrppZde0tq1a/Xhhx86s3QAKBOYR82BxRMo16xWq/773/+qRYsWCg8P16xZs2yXEFxdXSVJcXFxeumllyT9/k0T8+fPV2RkpCwWi9avX68OHTo48xAAwKmYR82FYIdy58rdA1cmHnd3d9WuXVtt2rRRdna2li1bJkm6dOmSunbtqk2bNikyMlLS75caIiIitGLFCr311lu2m4EBoCJhHjUv7rFDubFz505NnTpVAQEBioiI0K233ipJ+te//qXFixdr9uzZmjVrlpYsWSJvb2/NmDFDJ0+eVNOmTZ1cOQCUDcyj5scZO5QLq1at0kMPPaQ77rhD+/fv17PPPqt58+ZJkm655RY1adJEkrRu3Tr9+OOPyszMVO3atW2TEU9AB1DRMY9WDAQ7lAu7du3Svffeq+joaM2YMUNhYWFKSkrSr7/+qp9++knx8fG666675ObmprfeekuGYejXX3+1jecJ6AAqOubRioFVsSjzDMPQxYsXVadOHWVnZ6tatWrq0aOHfvvtNy1atEjPP/+8li9frueee04dOnTQkSNHlJOTY3siOgBUdMyjFQfBDmXalWcm1a1bV2vWrNGZM2dUrVo13XbbbQoODta6devk4uKif/zjH7Yx9erV05gxY5xYNQCUHcyjFQvnVVFmLFu2TKdOnZL0x4qtK89MGjp0qLKzs7VkyRJlZWVJkpo1a6aNGzfKze3330/y8/MlcbkAQMXFPAr+5lAmPPfcc3r88ce1aNEiSb9PRFcejnnFG2+8oaSkJM2YMUOStGfPHrVq1UqXL1+WJNvzlgCgImIehcTjTuBk//v1NL6+vjIMQ0888YSCgoJsfVJTUzVz5kzFx8dr5cqV+vzzz7V3716dP39eM2fO5MGYACo05lH8L87YwamunPZv0qSJ2rVrJ6vVqhUrVtjej4uLU3h4uO6++265ubnpwQcfVGJiohISErRt2zYmIwAVHvMo/hdn7FCq4uPjVbt2bTVr1kxBQUEyDENnz55VeHi4vvzyS61bt06ff/65qlSpou7duyskJEQ+Pj6qVKmSpD+ekg4AFRXzKK6Fv1mUipUrV6p9+/ZavXq1tm3bpk6dOikzM1MWi0VVq1ZVixYtVLVqVR06dEgrV67U6tWr1aJFCzVo0ECVKlXihl4AFR7zKBzBGTuUuOPHj2vUqFEaNmyY+vbtK0nq2bOn2rVrp9dff12HDx9Whw4d1LRpU50/f169evXSwYMHFRYWpkGDBjm5egBwPuZROIrn2KFEXLp0STk5OapWrZp8fHw0btw4tWnTxnYJoHHjxmrWrJkkyd/fXw8//LACAwMVHR0tSXrzzTfl6+vrzEMAAKdiHsWN4Iwdil18fLzeffddtWvXTl5eXpo9e7btvSsT0n333adnnnlGAwYMkPTHqq4/9wOAioh5FDeKv3EUq/Xr12vevHn69ttvFR8fr7S0NMXGxio9PV3S7/d27NmzR//9739tk1F6erosFovy8/Ntz1tiMgJQUTGP4mbwt46bdvbsWdufDx48qE6dOqlp06by8fHRu+++q9TUVH3//fe2Pvv371f//v117tw59e/fX88++6zy8vLk6upq99smAFQUzKMoLgQ73LC8vDyNHz9eYWFhevrpp/XNN9+oQYMGSkpKsvW577771LhxY23cuFFnzpyRJG3cuFHTp09Xt27d1LlzZy1ZssT2dTYAUJEwj6K4EexwQ7Zs2aJ27drp7Nmzmj17tu68804988wzuvfee3X58mV9+umntr6DBw/W8uXLbc9Q2rhxox5++GGtXbtWzz//vLMOAQCcinkUJYHFE7ghu3btUnJyskaPHi1J+u9//6uhQ4dqxowZ2rhxo55//nkdPXpU7u7ukqTu3btr6tSpCgoK0oULF1SlShVnlg8ATsc8ipJAsMMNy8rKUvXq1SVJe/fuVb9+/ZSSkqLq1avrgQceULVq1dS7d2+tXr1a6enp+uqrr1S5cmUnVw0AZQfzKIobl2Jxw65MRpK0e/du3X777ba2BQsW6KGHHlJSUpIaN26sVatWMRkBwJ8wj6K4caclbsqV5yT9+OOPateunSRp/vz5atiwocLDwzVgwADbZQQAQEHMoyhOnLHDTbnynKRdu3bJarUqIiJCCQkJtkmIyQgAro15FMWJe+xw0w4ePKiGDRuqbdu2GjlypB5//HFnlwQA5QrzKIoLwQ43LT8/X++9956ee+45eXh4OLscACh3mEdRXAh2AAAAJsE9dgAAACZBsAMAADAJgh0AAIBJEOwAAABMgmAHAABgEgQ7AAAAkyDYAQAAmATBDgCuYujQobJYLLJYLKpUqZJ8fX3Vo0cPzZkzR1ar1dnlAUABBDsAuIaePXvq2LFjOnDggL755ht169ZNo0ePVp8+fZSXl+fs8gDADsEOAK7Bw8NDfn5+atCggdq0aaOXX35Zy5cv1zfffKPExERJ0rRp03TXXXepatWq8vf311NPPaXs7GxJ0vnz5+Xl5aWlS5fafe6yZctUtWpVZWVllfYhATAxgh0AFNF9992nVq1a6YsvvpAkubi46P3339cvv/yiTz/9VGvXrtXYsWMlSVWrVtXAgQM1d+5cu8+YO3euHn74YVWvXr3U6wdgXnxXLABcxdChQ3X27FktW7aswHsDBw7UTz/9pF9//bXAe0uXLtWoUaN08uRJSdLmzZvVsWNHHT58WPXq1dPx48fVoEEDrV69Wl26dCnpwwBQgXDGDgBugGEYslgskqTVq1ere/fuatCggapXr64hQ4bo1KlTunDhgiSpffv2uvPOO/Xpp59KkubPn6+AgADde++9TqsfgDkR7ADgBuzYsUMNGzbUgQMH1KdPH7Vs2VL//Oc/lZqaqvj4eElSbm6urf/w4cNt9+TNnTtXw4YNswVDACguBDsAKKK1a9dq+/bteuihh5Samiqr1aqpU6fq7rvvVpMmTXT06NECY/72t7/p4MGDev/99/Xrr78qMjLSCZUDMDs3ZxcAAGVZTk6O0tPTlZ+fr4yMDCUnJys2NlZ9+vRRRESEfv75Z12+fFkffPCB+vbtq02bNikhIaHA59SsWVN/+ctf9OKLL+r+++/XLbfc4oSjAWB2nLEDgGtITk5WvXr1FBgYqJ49e2rdunV6//33tXz5crm6uqpVq1aaNm2a3n77bbVo0UILFixQbGxsoZ/1+OOPKzc3V4899lgpHwWAioJVsQBQSj777DONGTNGR48elbu7u7PLAWBCXIoFgBJ24cIFHTt2TG+99ZZGjhxJqANQYrgUCwAl7J133lGzZs3k5+enmJgYZ5cDwMS4FAsAAGASnLEDAAAwCYIdAACASRDsAAAATIJgBwAAYBIEOwAAAJMg2AEAAJgEwQ4AAMAkCHYAAAAmQbADAAAwif8HAAbe0vMZiM8AAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -337,8 +339,15 @@ ], "source": [ "# Visualize the cost by day\n", - "OpenAICostLoggerViz.plot_cost_by_day(path=DEFAULT_LOG_PATH)" + "OpenAICostLoggerViz.plot_cost_by_day(path=log_folder)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -357,7 +366,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/images/example.png b/images/example.png new file mode 100644 index 0000000..d1d7a38 Binary files /dev/null and b/images/example.png differ diff --git a/images/strftime_agg.png b/images/strftime_agg.png deleted file mode 100644 index ef23868..0000000 Binary files a/images/strftime_agg.png and /dev/null differ diff --git a/images/viz_prints.png b/images/viz_prints.png deleted file mode 100644 index 76153a7..0000000 Binary files a/images/viz_prints.png and /dev/null differ diff --git a/openai_cost_logger/__init__.py b/openai_cost_logger/__init__.py index 7a69ef6..e511bd8 100644 --- a/openai_cost_logger/__init__.py +++ b/openai_cost_logger/__init__.py @@ -1 +1,5 @@ +from .openai_cost_logger import OpenAICostLogger +from .constants import MODELS_COST, DEFAULT_LOG_PATH +from .openai_cost_logger_viz import OpenAICostLoggerViz +from .openai_cost_logger_utils import OpenAICostLoggerUtils print('imported openai_cost_logger') \ No newline at end of file diff --git a/openai_cost_logger/constants.py b/openai_cost_logger/constants.py index 6a72e32..c624b77 100644 --- a/openai_cost_logger/constants.py +++ b/openai_cost_logger/constants.py @@ -1,17 +1,8 @@ -from enum import Enum - """Default value for the cost-logs directory.""" DEFAULT_LOG_PATH = "cost-logs" + -"""Enum containing the tested models.""" -class Models(Enum): - TURBO_3_5 = "gpt-3.5-turbo" - TURBO_3_5_INSTRUCT = "gpt-3.5-turbo-instruct" - AZURE_3_5_TURBO = "gpt-35-turbo-0125" - AZURE_4_TURBO = "gpt-4-0125-Preview" - AZURE_4 = "gpt-4-0613" - -"""The costs of the models above (per million tokens).""" +"""The costs of the models above (per million tokens). Dictionary used in case the user does not provide the costs.""" MODELS_COST = { "gpt-3.5-turbo": { "input": 0.5, @@ -33,4 +24,8 @@ class Models(Enum): "input": 30, "output": 60 }, + "text-embedding-ada-002": { + "input": 0.1, + "output": 0.0 + } } \ No newline at end of file diff --git a/openai_cost_logger/openai_cost_logger.py b/openai_cost_logger/openai_cost_logger.py index 5bd64f0..8633f21 100644 --- a/openai_cost_logger/openai_cost_logger.py +++ b/openai_cost_logger/openai_cost_logger.py @@ -1,29 +1,21 @@ -import csv import json +import warnings from typing import Dict from pathlib import Path from time import strftime -from openai.types.chat.chat_completion import ChatCompletion +from openai._models import BaseModel # all the api responses extend BaseModel + +from openai_cost_logger.constants import DEFAULT_LOG_PATH, MODELS_COST -from openai_cost_logger.constants import DEFAULT_LOG_PATH """Every cost is per million tokens.""" COST_UNIT = 1_000_000 -"""Header of the cost log file.""" -FILE_HEADER = [ - "experiment_name", - "model", - "cost" -] """OpenAI cost logger.""" class OpenAICostLogger: def __init__( self, - model: str, - input_cost: float, - output_cost: float, experiment_name: str, cost_upperbound: float = float('inf'), log_folder: str = DEFAULT_LOG_PATH, @@ -32,21 +24,14 @@ def __init__( """Initialize the cost logger. Args: - client (enum.ClientType): The client to use. - model (str): The model to use. - cost_upperbound (float): The upperbound of the cost after which an exception is raised. - input_cost (float): The cost per million tokens for the input. - output_cost (float): The cost per million tokens for the output. experiment_name (str): The name of the experiment. + cost_upperbound (float): The upperbound of the cost after which an exception is raised. log_folder (str): The folder where to save the cost logs. - client_args (Dict, optional): The parameters to pass to the client. Defaults to {}. + log_level (str): The level of logging. # TODO: implement logging levels. """ self.cost = 0 self.n_responses = 0 - self.model = model - self.input_cost = input_cost self.log_folder = log_folder - self.output_cost = output_cost self.experiment_name = experiment_name self.cost_upperbound = cost_upperbound self.log_level = log_level @@ -58,16 +43,28 @@ def __init__( self.__build_log_file() - def update_cost(self, response: ChatCompletion) -> None: - """Extract the number of input and output tokens from a chat completion response - and update the cost. Saves experiment costs to file, overwriting it. + def update_cost(self, response: BaseModel, input_cost: float = None, output_cost: float = None) -> None: + """Extract the cost from the response and update the cost tracker. + Then write the cost to the json file for temporary storage. + Be aware that: + - the cost is calculated per million tokens. + - if input_cost and output_cost are not provided, the cost tracker will search for the values in the default dictionary. + In case the values are not found, the cost tracker will raise an exception. + Args: - response: ChatCompletion object from the model. + response (BaseModel): BaseModel object from the model. + input_cost (float, optional): The cost per million tokens for the input. Defaults to None. + output_cost (float, optional): The cost per million tokens for the output. Defaults to None. """ - self.cost += self.__get_answer_cost(response) + if (input_cost is None or output_cost is None) and response.model not in MODELS_COST: + raise Exception(f"Model {response.model} not found in the cost dictionary. Please provide the input and output cost.") + + input_cost = MODELS_COST[response.model]["input"] if input_cost is None else input_cost + output_cost = MODELS_COST[response.model]["output"] if output_cost is None else output_cost + self.cost += self.__get_answer_cost(response=response, input_cost=input_cost, output_cost=output_cost) self.n_responses += 1 - self.__write_cost_to_json(response) + self.__write_cost_to_json(response=response, input_cost=input_cost, output_cost=output_cost) self.__validate_cost() @@ -80,16 +77,24 @@ def get_current_cost(self) -> float: return self.cost - def __get_answer_cost(self, answer: Dict) -> float: - """Calculate the cost of the answer based on the input and output tokens. + def __get_answer_cost(self, response: BaseModel, input_cost: float, output_cost: float) -> float: + """Calculate the cost of the response based on the input and output tokens. Args: - answer (dict): The response from the model. + response (BaseModel): The response from the model. + input_cost (float): The cost per million tokens for the input. + output_cost (float): The cost per million tokens for the output. Returns: - float: The cost of the answer. + float: The cost of the answer. + Raises: + RuntimeWarning: If the output cost is 0 and there are completion tokens. """ - return (self.input_cost * answer.usage.prompt_tokens) / COST_UNIT + \ - (self.output_cost * answer.usage.completion_tokens) / COST_UNIT + completion_tokens = response.usage.total_tokens - response.usage.prompt_tokens + + if completion_tokens != 0 and output_cost == 0: + warnings.warn(f"Output cost: {output_cost}. Found {completion_tokens} completion tokens.", RuntimeWarning) + + return (input_cost * response.usage.prompt_tokens) / COST_UNIT + (output_cost * completion_tokens) / COST_UNIT def __validate_cost(self): @@ -98,21 +103,27 @@ def __validate_cost(self): Raises: Exception: If the cost exceeds the upperbound. """ - if self.cost > self.cost_upperbound: + if self.cost > self.cost_upperbound: raise Exception(f"Cost exceeded upperbound: {self.cost} > {self.cost_upperbound}") - def __write_cost_to_json(self, response: ChatCompletion) -> None: - """Write the cost to a json file. + def __write_cost_to_json(self, response: BaseModel, input_cost: float, output_cost: float) -> None: + """Write the cost to the json file. Args: - response (ChatCompletion): The response from the model. + response (BaseModel): The response from the model. + input_cost (float): The cost per million tokens for the input. + output_cost (float): The cost per million tokens for the output. """ with open(self.filepath, 'r') as file: data = json.load(file) data["total_cost"] = self.cost data["total_responses"] = self.n_responses - data["breakdown"].append(self.__build_log_breadown_entry(response)) + data["breakdown"].append(self.__build_log_breadown_entry( + response=response, + input_cost=input_cost, + output_cost=output_cost + )) with open(self.filepath, 'w') as file: json.dump(data, file, indent=4) @@ -127,7 +138,6 @@ def __build_log_file(self) -> None: log_file_template = { "experiment_name": self.experiment_name, "creation_datetime": strftime("%Y-%m-%d %H:%M:%S"), - "model": self.model, "total_cost": self.cost, "total_responses": 0, "breakdown": [] @@ -136,20 +146,28 @@ def __build_log_file(self) -> None: json.dump(log_file_template, file, indent=4) - def __build_log_breadown_entry(self, response: ChatCompletion) -> Dict: + def __build_log_breadown_entry(self, response: BaseModel, input_cost: float, output_cost: float) -> Dict: """Build a json log entry for the breakdown of the cost. + + Be aware that: + - The content of the response is supported only for the completion models. Args: - response (ChatCompletion): The response from the model. - + response (BaseModel): The response from the model. + input_cost (float): The cost per million tokens for the input. + output_cost (float): The cost per million tokens for the output. Returns: Dict: The json log entry. """ + output_tokens = response.usage.total_tokens - response.usage.prompt_tokens + content = response.choices[0].message.content if hasattr(response, "choices") else "content-not-supported-for-this-model" return { - "cost": self.__get_answer_cost(response), + "model": response.model, + "cost": self.__get_answer_cost(response=response, input_cost=input_cost, output_cost=output_cost), + "input_cost_per_million": input_cost, + "output_cost_per_million": output_cost, "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - "content": response.choices[0].message.content, - "inferred_model": response.model, + "output_tokens": output_tokens, + "content": content, "datetime": strftime("%Y-%m-%d %H:%M:%S"), } \ No newline at end of file diff --git a/openai_cost_logger/openai_cost_logger_utils.py b/openai_cost_logger/openai_cost_logger_utils.py index 362a8fc..c3e7a5d 100644 --- a/openai_cost_logger/openai_cost_logger_utils.py +++ b/openai_cost_logger/openai_cost_logger_utils.py @@ -1,10 +1,11 @@ from pathlib import Path + """OpenAI cost logger utilities functions.""" class OpenAICostLoggerUtils: @staticmethod - def get_api_key(path: str) -> str: + def read_api_key(path: str) -> str: """Return the key contained in the file. Args: diff --git a/openai_cost_logger/openai_cost_logger_viz.py b/openai_cost_logger/openai_cost_logger_viz.py index bfc3e24..dcf689e 100644 --- a/openai_cost_logger/openai_cost_logger_viz.py +++ b/openai_cost_logger/openai_cost_logger_viz.py @@ -1,13 +1,14 @@ import os import json -from datetime import datetime from typing import Dict from pathlib import Path +from datetime import datetime import matplotlib.pyplot as plt from collections import defaultdict from openai_cost_logger.constants import DEFAULT_LOG_PATH + """Cost logger visualizer.""" class OpenAICostLoggerViz: @@ -57,9 +58,8 @@ def get_total_cost_by_model(path: str = DEFAULT_LOG_PATH) -> Dict[str, float]: if filename.endswith(".json"): with open(Path(path, filename), mode='r') as file: data = json.load(file) - if data["model"] not in cost_by_model: - cost_by_model[data["model"]] = 0 - cost_by_model[data["model"]] += data["total_cost"] + for entry in data["breakdown"]: + cost_by_model[entry["model"]] += entry["cost"] return cost_by_model @@ -70,7 +70,7 @@ def print_total_cost_by_model(path: str = DEFAULT_LOG_PATH) -> None: log_folder (str, optional): Cost logs directory. Defaults to DEFAULT_LOG_PATH. This method reads all the files in the specified directory. """ - cost_by_model = OpenAICostLoggerViz.get_total_cost_by_model(path) + cost_by_model = OpenAICostLoggerViz.get_total_cost_by_model(path=path) for model, cost in cost_by_model.items(): print(f"{model}: {round(cost, 6)} (USD)") diff --git a/setup.py b/setup.py index 5e0ede3..797e609 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,10 @@ from setuptools import setup, find_packages -import os # Read the README file (reStructuredText format) with open('README.rst') as f: long_description = f.read() -# Get the version number from the environment -# version_number = os.getenv('VERSION_NUMBER') -# version_number = version_number.strip("v") - -version_number = '0.2.1' +version_number = '0.3.0' setup( name='openai-cost-logger', @@ -20,7 +15,7 @@ url='https://github.com/drudilorenzo/openai-cost-tracker', long_description=long_description, long_description_content_type='text/x-rst', - keywords=['openai', 'cost', 'logger', 'tracker'], + keywords=['openai', 'cost', 'logger', 'tracker', 'viz', 'llms'], license='MIT', packages=find_packages(include=['openai_cost_logger', 'openai_cost_logger.*']), requires=['openai', 'pandas', 'matplotlib'],