benchmark_llm.ipynb

{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:12:24.026469Z",
     "start_time": "2024-07-19T08:12:24.023057Z"
    }
   },
   "cell_type": "code",
   "source": "import os\n",
   "id": "c9c6d4c8e97993a7",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:12:25.822909Z",
     "start_time": "2024-07-19T08:12:24.045699Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from functools import reduce\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "def split_into_chunks(l, n):\n",
    "    \"\"\" split list into chunks with each of it being < n long. \n",
    "    source: https://www.geeksforgeeks.org/break-list-chunks-size-n-python \"\"\"\n",
    "    return [l[i * n:(i + 1) * n] for i in range((len(l) + n - 1) // n)]\n",
    "\n",
    "\n",
    "class Dataset:\n",
    "    def __init__(self, name, path, labels_files_names, is_dummy=False):\n",
    "        self.name = name\n",
    "        self.path = path\n",
    "        label_files_paths = self.get_labels_files_paths(path, labels_files_names)\n",
    "\n",
    "        self.schema = [\n",
    "            'projectname', 'packageandclass', 'path',\n",
    "            'readability', 'understandability', 'complexity',\n",
    "            'modularization', 'overall'\n",
    "        ]\n",
    "\n",
    "        self.dimensions = ['readability', 'understandability', 'complexity', 'modularization', 'overall']\n",
    "        self.dimensions_original = list(map(lambda d: d + '_original', self.dimensions))\n",
    "\n",
    "        self.dimensions_highest_probability_class = \\\n",
    "            list(map(lambda d: d + ' (highest probability class)', self.dimensions))\n",
    "\n",
    "        self.dimensions_highest_probability_class_simplified = \\\n",
    "            list(map(lambda d: d + ' (highest probability class simplified)', self.dimensions))\n",
    "\n",
    "        self.dimensions_classes_expectation_value = \\\n",
    "            list(map(lambda d: d + ' (class expectation value)', self.dimensions))\n",
    "\n",
    "        # class indices from 0 to 3:\n",
    "        self.classes = ['strongly agree', 'weakly agree', 'weakly disagree', 'strongly disagree']\n",
    "        # class indices implified (binary):\n",
    "        self.classes_simplified = ['agree', 'disagree']\n",
    "\n",
    "        # read labels files\n",
    "        column_indices = [self.schema.index(col) for col in ['path', *self.dimensions]]\n",
    "        self.dataframe = self.read_labels_file(label_files_paths.pop(0), column_indices)\n",
    "        while len(label_files_paths) > 0:\n",
    "            further_dataframe = self.read_labels_file(label_files_paths.pop(0), column_indices)\n",
    "            self.dataframe = self.dataframe.append(further_dataframe, ignore_index=True)\n",
    "\n",
    "        # change dimension column names to dimensionname_original\n",
    "        self.dataframe = self.dataframe.rename(columns=dict(zip(self.dimensions, self.dimensions_original)))\n",
    "\n",
    "        # read code from file path as string\n",
    "        self.dataframe['code'] = self.dataframe['path'].apply(self.read_code_file)\n",
    "\n",
    "        self.set_class_interpretations()\n",
    "\n",
    "    def get_labels_files_paths(self, path, labels_files_names):\n",
    "        return [os.path.join(path, label_file_name) for label_file_name in labels_files_names]\n",
    "\n",
    "    def read_labels_file(self, file_path, chosen_column_indices):\n",
    "        return pd.read_csv(file_path, sep=',', usecols=chosen_column_indices)\n",
    "\n",
    "    def read_code_file(self, file_path):\n",
    "        file_path = os.path.join(self.path, *file_path.split('\\\\'))\n",
    "        file_content = open(file_path, 'r', encoding='ISO-8859-1').read()\n",
    "        return file_content\n",
    "\n",
    "    def set_class_interpretations(self):\n",
    "        for i in range(len(self.dimensions)):\n",
    "            # Take the argmax of the Likert scale ratings, thus resulting in 4 possible discrete classes \n",
    "            # (strongly agree, weakly agree, weakly disagree, strongly disagree) => the dataset is very imbalanced\n",
    "            self.dataframe[self.dimensions_highest_probability_class[i]] = \\\n",
    "                self.dataframe[self.dimensions_original[i]].apply(self.get_index_highest_prob_class)\n",
    "\n",
    "            # Binary approach to cope with imbalance: the 3 least represented classes are merged into 1 class, \n",
    "            # such that the result are 2 discrete classes (agree, disagree) that are roughly balanced\n",
    "            self.dataframe[self.dimensions_highest_probability_class_simplified[i]] = \\\n",
    "                self.dataframe[self.dimensions_highest_probability_class[i]].apply(\n",
    "                    lambda c: self.get_index_highest_prob_class_simplified(c, self.dimensions[i]))\n",
    "\n",
    "            # Calculate the expectation value of the given probabilites for the Likert scale ratings\n",
    "            self.dataframe[self.dimensions_classes_expectation_value[i]] = \\\n",
    "                self.dataframe[self.dimensions_original[i]].apply(self.get_class_expectation_value)\n",
    "\n",
    "    def get_index_highest_prob_class(self, class_probabilities_string):\n",
    "        # calculate the index of the class with the highest probability\n",
    "        probabilities = self.parse_class_probabilities(class_probabilities_string)\n",
    "        index = np.argmax(probabilities)\n",
    "        return index\n",
    "\n",
    "    def get_index_highest_prob_class_simplified(self, class_index, dimension):\n",
    "        # similar to get_index_highest_prob_class; this time merging the 3 least popular classes into 1 class;\n",
    "        # for complexity, the first 3 classes are the minority, for the other dimensions the last 3 classes\n",
    "        agree_indices = [0, 1, 2] if dimension == 'complexity' else [0]\n",
    "        # based on self.classes_simplified\n",
    "        binary_class = 0 if class_index in agree_indices else 1\n",
    "        return binary_class\n",
    "\n",
    "    def get_class_expectation_value(self, class_probabilities_string):\n",
    "        # calculate the expectation value of the classes\n",
    "        indices = range(1, len(self.classes) + 1)\n",
    "        probabilities = self.parse_class_probabilities(class_probabilities_string)\n",
    "        expected_value = reduce(lambda acc, t: acc + t[0] * t[1], zip(indices, probabilities), 0)\n",
    "        return expected_value\n",
    "\n",
    "    def parse_class_probabilities(self, class_probabilities_string):\n",
    "        # class_probabilities_string looks like this: {4.38897e-007,0.848266,0.151734,3.13027e-008}\n",
    "        number_strings = class_probabilities_string.replace('{', '').replace('}', '').split(',')\n",
    "        probabilities = list(map(float, number_strings))\n",
    "        return probabilities\n",
    "\n",
    "    def shuffle(self, SEED=42):\n",
    "        self.dataframe = self.dataframe.sample(frac=1, random_state=SEED).reset_index(drop=True)\n",
    "\n",
    "    def stratify_split(self, frac_training, frac_validation, frac_test, stratify_col_name):\n",
    "        \"\"\" adapted from https://stackoverflow.com/a/65571687 \"\"\"\n",
    "\n",
    "        x = self.dataframe  # Contains all columns\n",
    "        y = self.dataframe[[stratify_col_name]]  # Dataframe of just the column on which to stratify\n",
    "\n",
    "        # Split original dataframe into train and temp dataframes\n",
    "        df_training, df_temp, y_train, y_temp = \\\n",
    "            train_test_split(x, y, stratify=y, test_size=(1.0 - frac_training), random_state=None)\n",
    "\n",
    "        if frac_validation == 0.0:\n",
    "            return df_training, df_temp\n",
    "        else:\n",
    "            # Split the temp dataframe into val and test dataframes\n",
    "            relative_frac_test = frac_test / (frac_validation + frac_test)\n",
    "            df_validation, df_test, y_val, y_test = \\\n",
    "                train_test_split(df_temp, y_temp, stratify=y_temp, test_size=relative_frac_test, random_state=None)\n",
    "\n",
    "            return df_training, df_validation, df_test\n",
    "\n",
    "    def bert_tokenize(self, tokenizer, target_column_name):\n",
    "        # if already tokenized, don't do it again\n",
    "        if target_column_name in self.dataframe.columns:\n",
    "            return\n",
    "\n",
    "        print('bert tokenization...')\n",
    "\n",
    "        # work in another dataframe as a workaround to use object data type\n",
    "        encoding_temp = pd.DataFrame(columns=[target_column_name])\n",
    "        encoding_temp[target_column_name] = encoding_temp[target_column_name].astype(object)\n",
    "        # input is limited to 512 tokens, therefore we split up each code into chunks \n",
    "        # and keep track of all chunks including the other information in a new dataframe\n",
    "        dataframe_extended = pd.DataFrame().reindex_like(self.dataframe)\n",
    "        dataframe_extended_current_length = 0\n",
    "        for index, row in self.dataframe.iterrows():\n",
    "            tokens = tokenizer.tokenize(row['code'])\n",
    "            tokens_chunks = split_into_chunks(tokens, 506)  # leave space for special chars\n",
    "            for tokens in tokens_chunks:\n",
    "                correctly_sized_string = tokenizer.convert_tokens_to_string(tokens)\n",
    "                encoding = tokenizer.encode_plus(correctly_sized_string, padding='max_length')\n",
    "                dataframe_extended.loc[dataframe_extended_current_length] = row\n",
    "                encoding_temp.loc[dataframe_extended_current_length, target_column_name] = np.array(encoding.input_ids)\n",
    "                dataframe_extended_current_length += 1\n",
    "        dataframe_extended[target_column_name] = encoding_temp[target_column_name]\n",
    "\n",
    "        # somehow it appends \".0\" to the old floats, which we need to revert:\n",
    "        for dimension in self.dimensions_highest_probability_class + self.dimensions_highest_probability_class_simplified:\n",
    "            dataframe_extended[dimension] = dataframe_extended[dimension].astype(int)\n",
    "\n",
    "        self.dataframe = dataframe_extended\n",
    "\n",
    "        print(self.dataframe)\n",
    "\n",
    "    def print_statistics(self, show_scatter_plot=False):\n",
    "        # dataframe\n",
    "        print(self.dataframe)\n",
    "\n",
    "        # number of times each class occurs per dimension\n",
    "        for dimension in self.dimensions_highest_probability_class + self.dimensions_highest_probability_class_simplified:\n",
    "            print(self.dataframe[dimension].value_counts(), '\\n')\n",
    "\n",
    "        # scatter plot for regression variant\n",
    "        if show_scatter_plot:\n",
    "            mpl.rcParams['figure.dpi'] = 300\n",
    "            plt.figure(figsize=(4, 10))\n",
    "            plt.xticks(range(len(self.dimensions)), self.dimensions)\n",
    "            plt.yticks(range(len(self.classes)), self.classes)\n",
    "            plt.gca().invert_yaxis()\n",
    "            for index, dimension in enumerate(self.dimensions_classes_expectation_value):\n",
    "                values = self.dataframe[dimension]\n",
    "                plt.plot(np.zeros_like(values) + index, values, '_')\n",
    "            plt.show()\n",
    "\n",
    "\n",
    "def print_section(string):\n",
    "    print('==========================\\n', string, '\\n==========================')\n",
    "\n",
    "\n",
    "# dataset setups\n",
    "dataset_path = \"/mnt/c/Data/Apps/hackathon/hackjakarta/playground/dataset\"\n",
    "datasets = [Dataset('open source dataset', dataset_path, ['labels.csv'])]"
   ],
   "id": "b3bcc7b84eb43a6e",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:12:25.835543Z",
     "start_time": "2024-07-19T08:12:25.824009Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "# iterate over dataset setups (open source parts, all parts)\n",
    "for dataset in datasets:\n",
    "    print_section('dataset: ' + dataset.name)\n",
    "    dataset.shuffle()\n",
    "    dataset.print_statistics()\n",
    "\n",
    "    # iterate over dataset interpretations (Likert scale and binary)\n",
    "    for dimensions, classes in [\n",
    "        (dataset.dimensions_highest_probability_class, dataset.classes),\n",
    "        (dataset.dimensions_highest_probability_class_simplified, dataset.classes_simplified)]:\n",
    "\n",
    "        # iterate over dimensions\n",
    "        for dimension in dimensions:\n",
    "            print_section('dimension: ' + dimension)\n",
    "            print(dataset.dataframe[dimension].value_counts(), '\\n')"
   ],
   "id": "c96bcbfa5433c6b7",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      " dataset: open source dataset \n",
      "==========================\n",
      "                                                  path  \\\n",
      "0    jsweet\\sourcefiles\\core-lib\\es6\\src\\main\\java\\...   \n",
      "1    diarymanagement\\sourcefiles\\tbaclient\\src\\TBA\\...   \n",
      "2    argoUML\\sourcefiles\\src\\argouml-app\\src\\org\\ar...   \n",
      "3    junit4\\sourcefiles\\src\\main\\java\\org\\junit\\exp...   \n",
      "4    aoi\\sourcefiles\\Translators\\src\\artofillusion\\...   \n",
      "..                                                 ...   \n",
      "299  jsweet\\sourcefiles\\core-lib\\es6\\src\\main\\java\\...   \n",
      "300   aoi\\sourcefiles\\HelpPlugin\\src\\ChatTextPane.java   \n",
      "301  argoUML\\sourcefiles\\src\\argouml-app\\src\\org\\ar...   \n",
      "302  junit4\\sourcefiles\\src\\main\\java\\org\\junit\\int...   \n",
      "303  argoUML\\sourcefiles\\src\\argouml-app\\src\\org\\ar...   \n",
      "\n",
      "                              readability_original  \\\n",
      "0      {0.971194,0.0288058,3.07528e-007,8.37e-018}   \n",
      "1                             {1,1.04008e-011,0,0}   \n",
      "2       {0.029332,0.954998,0.0156261,4.43023e-005}   \n",
      "3    {0.788656,0.211336,7.75132e-006,1.19846e-011}   \n",
      "4    {1.96205e-005,0.885832,0.114149,8.07257e-009}   \n",
      "..                                             ...   \n",
      "299              {0.99726,0.00274036,7.272e-017,0}   \n",
      "300  {1.00053e-011,0.95712,0.0428768,3.59289e-006}   \n",
      "301                {0.999872,0.000127839,3e-020,0}   \n",
      "302    {1.89398e-009,0.23087,0.76913,8.51906e-008}   \n",
      "303  {0.909891,0.090104,3.24958e-006,1.24981e-006}   \n",
      "\n",
      "                        understandability_original  \\\n",
      "0    {0.888599,0.111397,3.26292e-006,5.89971e-015}   \n",
      "1                      {0.999992,8.02978e-006,0,0}   \n",
      "2        {0.000188948,0.276109,0.707136,0.0165661}   \n",
      "3        {0.307977,0.68612,0.0059027,3.93822e-011}   \n",
      "4       {9.81071e-009,0.0733606,0.613897,0.312742}   \n",
      "..                                             ...   \n",
      "299             {0.769791,0.230209,2.52692e-007,0}   \n",
      "300      {9.93292e-013,0.322445,0.63569,0.0418648}   \n",
      "301       {0.00105809,0.998935,6.4894e-006,1e-020}   \n",
      "302    {0.270327,0.721304,0.00836896,2.76098e-010}   \n",
      "303     {0.576995,0.42189,0.00111496,1.03004e-010}   \n",
      "\n",
      "                               complexity_original  \\\n",
      "0          {9e-020,1.1399e-007,0.0242425,0.975757}   \n",
      "1                             {0,0,2.94932e-012,1}   \n",
      "2       {0.0077364,0.392712,0.599551,6.74932e-008}   \n",
      "3    {1.66225e-007,0.000676127,0.0373025,0.962021}   \n",
      "4        {0.107004,0.84513,0.0478661,1.25577e-012}   \n",
      "..                                             ...   \n",
      "299                    {0,3.4e-019,1.95389e-007,1}   \n",
      "300         {0.458368,0.15113,0.387471,0.00303148}   \n",
      "301  {1.38117e-007,0.895009,0.104991,4.90838e-008}   \n",
      "302      {0.0504081,0.754762,0.19483,9.32982e-009}   \n",
      "303    {1.705e-012,2.96941e-005,0.90928,0.0906898}   \n",
      "\n",
      "                           modularization_original  \\\n",
      "0              {0,4.28517e-011,0.0203369,0.979663}   \n",
      "1                            {0,0,0.02413,0.97587}   \n",
      "2      {5.54443e-007,0.168712,0.829111,0.00217636}   \n",
      "3    {2.47902e-006,0.161017,0.838981,5.11241e-008}   \n",
      "4              {0,2.49883e-014,0.99519,0.00481021}   \n",
      "..                                             ...   \n",
      "299             {0,5.74298e-011,0.127985,0.872015}   \n",
      "300                       {0.99974,0.00025978,0,0}   \n",
      "301          {0,7.21282e-015,0.999675,0.000325138}   \n",
      "302            {0,2.65974e-009,0.0958948,0.904105}   \n",
      "303            {0,3.02134e-011,0.0140928,0.985907}   \n",
      "\n",
      "                               overall_original  \\\n",
      "0           {0.986171,0.0138287,9.98904e-012,0}   \n",
      "1     {1,4.19334e-007,1.16448e-009,8.6548e-016}   \n",
      "2    {0.0011249,0.721043,0.277832,3.41147e-007}   \n",
      "3           {0.0721647,0.927832,3.72261e-006,0}   \n",
      "4    {2.16648e-008,0.160153,0.813925,0.0259221}   \n",
      "..                                          ...   \n",
      "299          {0.814165,0.185835,1.25247e-010,0}   \n",
      "300  {5.19486e-010,0.0402193,0.717704,0.242077}   \n",
      "301         {7.80593e-005,0.990595,0.0093268,0}   \n",
      "302   {0.157794,0.842188,1.8479e-005,1.21e-018}   \n",
      "303        {0.993417,0.00658346,7.92256e-012,0}   \n",
      "\n",
      "                                                  code  \\\n",
      "0    package def.dom.intl;\\n@jsweet.lang.Interface\\...   \n",
      "1    package TBA.Messages;\\n\\nimport TBA.Exceptions...   \n",
      "2    /* $Id: ZoomSliderButton.java 19907 2012-12-30...   \n",
      "3    package org.junit.experimental.max;\\n\\nimport ...   \n",
      "4    /* Copyright (C) 2003 by Peter Eastman\\n\\n   T...   \n",
      "..                                                 ...   \n",
      "299  package def.dom;\\n\\nimport def.js.Object;\\nimp...   \n",
      "300  /*  ChatTextPane.java  */\\n\\npackage nik777.ch...   \n",
      "301  /* $Id: ShortcutField.java 17842 2010-01-12 19...   \n",
      "302  package org.junit.internal.runners;\\n\\nimport ...   \n",
      "303  /* $Id: AssociationEndNameNotationJava.java 18...   \n",
      "\n",
      "     readability (highest probability class)  \\\n",
      "0                                          0   \n",
      "1                                          0   \n",
      "2                                          1   \n",
      "3                                          0   \n",
      "4                                          1   \n",
      "..                                       ...   \n",
      "299                                        0   \n",
      "300                                        1   \n",
      "301                                        0   \n",
      "302                                        2   \n",
      "303                                        0   \n",
      "\n",
      "     readability (highest probability class simplified)  \\\n",
      "0                                                    0    \n",
      "1                                                    0    \n",
      "2                                                    1    \n",
      "3                                                    0    \n",
      "4                                                    1    \n",
      "..                                                 ...    \n",
      "299                                                  0    \n",
      "300                                                  1    \n",
      "301                                                  0    \n",
      "302                                                  1    \n",
      "303                                                  0    \n",
      "\n",
      "     readability (class expectation value)  ...  \\\n",
      "0                                 1.028807  ...   \n",
      "1                                 1.000000  ...   \n",
      "2                                 1.986384  ...   \n",
      "3                                 1.211351  ...   \n",
      "4                                 2.114131  ...   \n",
      "..                                     ...  ...   \n",
      "299                               1.002741  ...   \n",
      "300                               2.042885  ...   \n",
      "301                               1.000128  ...   \n",
      "302                               2.769130  ...   \n",
      "303                               1.090114  ...   \n",
      "\n",
      "     understandability (class expectation value)  \\\n",
      "0                                       1.111403   \n",
      "1                                       1.000008   \n",
      "2                                       2.740079   \n",
      "3                                       1.697925   \n",
      "4                                       3.239380   \n",
      "..                                           ...   \n",
      "299                                     1.230210   \n",
      "300                                     2.719419   \n",
      "301                                     1.998948   \n",
      "302                                     1.738042   \n",
      "303                                     1.424120   \n",
      "\n",
      "     complexity (highest probability class)  \\\n",
      "0                                         3   \n",
      "1                                         3   \n",
      "2                                         2   \n",
      "3                                         3   \n",
      "4                                         1   \n",
      "..                                      ...   \n",
      "299                                       3   \n",
      "300                                       0   \n",
      "301                                       1   \n",
      "302                                       1   \n",
      "303                                       2   \n",
      "\n",
      "     complexity (highest probability class simplified)  \\\n",
      "0                                                    1   \n",
      "1                                                    1   \n",
      "2                                                    0   \n",
      "3                                                    1   \n",
      "4                                                    0   \n",
      "..                                                 ...   \n",
      "299                                                  1   \n",
      "300                                                  0   \n",
      "301                                                  0   \n",
      "302                                                  0   \n",
      "303                                                  0   \n",
      "\n",
      "     complexity (class expectation value)  \\\n",
      "0                                3.975756   \n",
      "1                                4.000000   \n",
      "2                                2.591814   \n",
      "3                                3.961344   \n",
      "4                                1.940862   \n",
      "..                                    ...   \n",
      "299                              4.000001   \n",
      "300                              1.935167   \n",
      "301                              2.104991   \n",
      "302                              2.144422   \n",
      "303                              3.090659   \n",
      "\n",
      "     modularization (highest probability class)  \\\n",
      "0                                             3   \n",
      "1                                             3   \n",
      "2                                             2   \n",
      "3                                             2   \n",
      "4                                             2   \n",
      "..                                          ...   \n",
      "299                                           3   \n",
      "300                                           0   \n",
      "301                                           2   \n",
      "302                                           3   \n",
      "303                                           3   \n",
      "\n",
      "     modularization (highest probability class simplified)  \\\n",
      "0                                                    1       \n",
      "1                                                    1       \n",
      "2                                                    1       \n",
      "3                                                    1       \n",
      "4                                                    1       \n",
      "..                                                 ...       \n",
      "299                                                  1       \n",
      "300                                                  0       \n",
      "301                                                  1       \n",
      "302                                                  1       \n",
      "303                                                  1       \n",
      "\n",
      "     modularization (class expectation value)  \\\n",
      "0                                    3.979663   \n",
      "1                                    3.975870   \n",
      "2                                    2.833463   \n",
      "3                                    2.838980   \n",
      "4                                    3.004811   \n",
      "..                                        ...   \n",
      "299                                  3.872015   \n",
      "300                                  1.000260   \n",
      "301                                  3.000326   \n",
      "302                                  3.904104   \n",
      "303                                  3.985906   \n",
      "\n",
      "     overall (highest probability class)  \\\n",
      "0                                      0   \n",
      "1                                      0   \n",
      "2                                      1   \n",
      "3                                      1   \n",
      "4                                      2   \n",
      "..                                   ...   \n",
      "299                                    0   \n",
      "300                                    2   \n",
      "301                                    1   \n",
      "302                                    1   \n",
      "303                                    0   \n",
      "\n",
      "     overall (highest probability class simplified)  \\\n",
      "0                                                 0   \n",
      "1                                                 0   \n",
      "2                                                 1   \n",
      "3                                                 1   \n",
      "4                                                 1   \n",
      "..                                              ...   \n",
      "299                                               0   \n",
      "300                                               1   \n",
      "301                                               1   \n",
      "302                                               1   \n",
      "303                                               0   \n",
      "\n",
      "     overall (class expectation value)  \n",
      "0                             1.013828  \n",
      "1                             1.000001  \n",
      "2                             2.276708  \n",
      "3                             1.927840  \n",
      "4                             2.865769  \n",
      "..                                 ...  \n",
      "299                           1.185835  \n",
      "300                           3.201859  \n",
      "301                           2.009248  \n",
      "302                           1.842225  \n",
      "303                           1.006584  \n",
      "\n",
      "[304 rows x 22 columns]\n",
      "readability (highest probability class)\n",
      "0    183\n",
      "1     79\n",
      "2     38\n",
      "3      4\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "understandability (highest probability class)\n",
      "0    157\n",
      "1     76\n",
      "2     51\n",
      "3     20\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "complexity (highest probability class)\n",
      "3    181\n",
      "2     60\n",
      "1     41\n",
      "0     22\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "modularization (highest probability class)\n",
      "3    195\n",
      "2     49\n",
      "1     31\n",
      "0     29\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "overall (highest probability class)\n",
      "0    174\n",
      "1     64\n",
      "2     41\n",
      "3     25\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "readability (highest probability class simplified)\n",
      "0    183\n",
      "1    121\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "understandability (highest probability class simplified)\n",
      "0    157\n",
      "1    147\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "complexity (highest probability class simplified)\n",
      "1    181\n",
      "0    123\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "modularization (highest probability class simplified)\n",
      "1    275\n",
      "0     29\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "overall (highest probability class simplified)\n",
      "0    174\n",
      "1    130\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: readability (highest probability class) \n",
      "==========================\n",
      "readability (highest probability class)\n",
      "0    183\n",
      "1     79\n",
      "2     38\n",
      "3      4\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: understandability (highest probability class) \n",
      "==========================\n",
      "understandability (highest probability class)\n",
      "0    157\n",
      "1     76\n",
      "2     51\n",
      "3     20\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: complexity (highest probability class) \n",
      "==========================\n",
      "complexity (highest probability class)\n",
      "3    181\n",
      "2     60\n",
      "1     41\n",
      "0     22\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: modularization (highest probability class) \n",
      "==========================\n",
      "modularization (highest probability class)\n",
      "3    195\n",
      "2     49\n",
      "1     31\n",
      "0     29\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: overall (highest probability class) \n",
      "==========================\n",
      "overall (highest probability class)\n",
      "0    174\n",
      "1     64\n",
      "2     41\n",
      "3     25\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: readability (highest probability class simplified) \n",
      "==========================\n",
      "readability (highest probability class simplified)\n",
      "0    183\n",
      "1    121\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: understandability (highest probability class simplified) \n",
      "==========================\n",
      "understandability (highest probability class simplified)\n",
      "0    157\n",
      "1    147\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: complexity (highest probability class simplified) \n",
      "==========================\n",
      "complexity (highest probability class simplified)\n",
      "1    181\n",
      "0    123\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: modularization (highest probability class simplified) \n",
      "==========================\n",
      "modularization (highest probability class simplified)\n",
      "1    275\n",
      "0     29\n",
      "Name: count, dtype: int64 \n",
      "\n",
      "==========================\n",
      " dimension: overall (highest probability class simplified) \n",
      "==========================\n",
      "overall (highest probability class simplified)\n",
      "0    174\n",
      "1    130\n",
      "Name: count, dtype: int64 \n",
      "\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:12:25.838344Z",
     "start_time": "2024-07-19T08:12:25.836425Z"
    }
   },
   "cell_type": "code",
   "source": "dataset = datasets[0]",
   "id": "340ce0162be003c4",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:12:25.997346Z",
     "start_time": "2024-07-19T08:12:25.986947Z"
    }
   },
   "cell_type": "code",
   "source": "dataset.dataframe[dataset.dimensions_classes_expectation_value].describe()",
   "id": "21f8861ce7c43981",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       readability (class expectation value)  \\\n",
       "count                             304.000000   \n",
       "mean                                1.574479   \n",
       "std                                 0.715332   \n",
       "min                                 1.000000   \n",
       "25%                                 1.042815   \n",
       "50%                                 1.099956   \n",
       "75%                                 2.002161   \n",
       "max                                 3.999755   \n",
       "\n",
       "       understandability (class expectation value)  \\\n",
       "count                                   304.000000   \n",
       "mean                                      1.806059   \n",
       "std                                       0.909919   \n",
       "min                                       1.000000   \n",
       "25%                                       1.052075   \n",
       "50%                                       1.355439   \n",
       "75%                                       2.255207   \n",
       "max                                       4.000000   \n",
       "\n",
       "       complexity (class expectation value)  \\\n",
       "count                            304.000000   \n",
       "mean                               3.286172   \n",
       "std                                0.920122   \n",
       "min                                1.000000   \n",
       "25%                                2.926884   \n",
       "50%                                3.900417   \n",
       "75%                                3.950188   \n",
       "max                                4.000001   \n",
       "\n",
       "       modularization (class expectation value)  \\\n",
       "count                                304.000000   \n",
       "mean                                   3.314164   \n",
       "std                                    0.947128   \n",
       "min                                    1.000000   \n",
       "25%                                    2.992522   \n",
       "50%                                    3.888660   \n",
       "75%                                    3.960932   \n",
       "max                                    4.000001   \n",
       "\n",
       "       overall (class expectation value)  \n",
       "count                         304.000000  \n",
       "mean                            1.755370  \n",
       "std                             0.955734  \n",
       "min                             1.000000  \n",
       "25%                             1.016285  \n",
       "50%                             1.117790  \n",
       "75%                             2.153418  \n",
       "max                             4.000000  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>readability (class expectation value)</th>\n",
       "      <th>understandability (class expectation value)</th>\n",
       "      <th>complexity (class expectation value)</th>\n",
       "      <th>modularization (class expectation value)</th>\n",
       "      <th>overall (class expectation value)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>304.000000</td>\n",
       "      <td>304.000000</td>\n",
       "      <td>304.000000</td>\n",
       "      <td>304.000000</td>\n",
       "      <td>304.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.574479</td>\n",
       "      <td>1.806059</td>\n",
       "      <td>3.286172</td>\n",
       "      <td>3.314164</td>\n",
       "      <td>1.755370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.715332</td>\n",
       "      <td>0.909919</td>\n",
       "      <td>0.920122</td>\n",
       "      <td>0.947128</td>\n",
       "      <td>0.955734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.042815</td>\n",
       "      <td>1.052075</td>\n",
       "      <td>2.926884</td>\n",
       "      <td>2.992522</td>\n",
       "      <td>1.016285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.099956</td>\n",
       "      <td>1.355439</td>\n",
       "      <td>3.900417</td>\n",
       "      <td>3.888660</td>\n",
       "      <td>1.117790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.002161</td>\n",
       "      <td>2.255207</td>\n",
       "      <td>3.950188</td>\n",
       "      <td>3.960932</td>\n",
       "      <td>2.153418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.999755</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000001</td>\n",
       "      <td>4.000001</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:32:26.785652Z",
     "start_time": "2024-07-19T08:32:26.782951Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from pydantic.v1 import BaseModel, Field\n",
    "\n",
    "\n",
    "class CodeScoring(BaseModel):\n",
    "    \"\"\"\n",
    "    Score code in based on the following statements: \n",
    "    Readability - this code is easy to read; \n",
    "    Understandability - the semantic meaning of this code is clear; \n",
    "    Complexity - this code is complex; \n",
    "    Modularity  - this code should be broken into smaller pieces; \n",
    "    Overall maintainability - overall, this code is maintainable. \n",
    "    The score scale from -1000000 (strongly agree) to 1000000 (strongly disagree).\n",
    "    You must score in precision, i.e. 14952, -475.456, 757850, 584.58495, 3.141598, etc.\n",
    "    \"\"\"\n",
    "    readability: float = Field(description=\"Readability score.\")\n",
    "    understandability: float = Field(description=\"Understandability score.\")\n",
    "    complexity: float = Field(description=\"Complexity score.\")\n",
    "    modularization: float = Field(description=\"Modularity score.\")\n",
    "    overall_maintainability: float = Field(description=\"Overall maintainability score.\")\n"
   ],
   "id": "5055ce504ab4b274",
   "outputs": [],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:39:02.476831Z",
     "start_time": "2024-07-19T08:39:02.372147Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from langchain_core.runnables import RunnableParallel\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain_core.messages import AIMessage\n",
    "from typing import TypedDict\n",
    "from langgraph.graph import StateGraph, MessagesState\n",
    "from langchain_core.output_parsers import PydanticToolsParser\n",
    "from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate, \\\n",
    "    AIMessagePromptTemplate\n",
    "\n",
    "chat = ChatOpenAI(\n",
    "    model=\"gpt-4o-mini\",\n",
    "    api_key=OPENAI_API_KEY\n",
    ")\n",
    "\n",
    "tools = [CodeScoring]\n",
    "parser = PydanticToolsParser(tools=tools)\n",
    "\n",
    "\n",
    "class State(TypedDict):\n",
    "    code: str\n",
    "    analysis: str\n",
    "    score: CodeScoring\n",
    "\n",
    "\n",
    "def node_analyze(state: State):\n",
    "    prompt = ChatPromptTemplate.from_messages([\n",
    "        SystemMessagePromptTemplate.from_template(\"\"\"\n",
    "    You are target group of our study. The target group of our study are software quality analysts, researchers with a background in software quality, and software engineers that are involved with maintaining software. Some participants have up to 15 years of experience in quality assessments. In sum, 70 professionals participated. First, we invited selected experts to participate in the study. Second, we asked them to disseminate the study to interested and qualified colleagues. The survey was also promoted in relevant net- works. The participants are affiliated with companies including Airbus, Audi, BMW, Boston Consulting Group, Celonis, cesdo Software Quality GmbH, CQSE GmbH, Facebook, fortiss, itestra GmbH, Kinexon GmbH, MaibornWolff GmbH, Munich Re, Oracle, and three universities. However, 7 participants did not want to share their affiliation. During the study, we followed a systematic approach to assess software maintainability. The following steps were taken:\n",
    "    \"\"\"),\n",
    "        HumanMessagePromptTemplate.from_template(\"\"\"\n",
    "    Analyze readability, understandability, complexity, modularity, and overall maintainability metrics of the following code.\n",
    "    You must use step by step comprehensive reasoning to explain your analysis.\n",
    "    <code>\n",
    "    {code}\n",
    "    </code>\n",
    "    \"\"\"),\n",
    "    ])\n",
    "    chain = prompt | chat\n",
    "    response = chain.invoke({\n",
    "        \"code\": state[\"code\"],\n",
    "    })\n",
    "    state[\"analysis\"] = response.content\n",
    "    return state\n",
    "\n",
    "\n",
    "def node_score(state: State):\n",
    "    prompt = ChatPromptTemplate.from_messages([\n",
    "        SystemMessagePromptTemplate.from_template(\"\"\"\n",
    "        You are target group of our study. The target group of our study are software quality analysts, researchers with a background in software quality, and software engineers that are involved with maintaining software. Some participants have up to 15 years of experience in quality assessments. In sum, 70 professionals participated. First, we invited selected experts to participate in the study. Second, we asked them to disseminate the study to interested and qualified colleagues. The survey was also promoted in relevant networks. The participants are affiliated with companies including Airbus, Audi, BMW, Boston Consulting Group, Celonis, cesdo Software Quality GmbH, CQSE GmbH, Facebook, fortiss, itestra GmbH, Kinexon GmbH, MaibornWolff GmbH, Munich Re, Oracle, and three universities. However, 7 participants did not want to share their affiliation. During the study, we followed a systematic approach to assess software maintainability. The following steps were taken:\n",
    "        \"\"\"),\n",
    "        HumanMessagePromptTemplate.from_template(\"\"\"\n",
    "        Analyze readability, understandability, complexity, modularity, and overall maintainability metrics of the following code.\n",
    "        You must use step by step comprehensive reasoning to explain your analysis.\n",
    "        <code>\n",
    "        {code}\n",
    "        </code>\n",
    "        \"\"\"),\n",
    "        AIMessagePromptTemplate.from_template(\"\"\"\n",
    "        {analysis}\n",
    "        \"\"\"),\n",
    "        HumanMessagePromptTemplate.from_template(\"\"\"\n",
    "        Based on your analysis and the provided code, score the code based on the following criteria:\n",
    "        Readability - this code is easy to read; \n",
    "        Understandability - the semantic meaning of this code is clear; \n",
    "        Complexity - this code is complex; \n",
    "        Modularity  - this code should be broken into smaller pieces; \n",
    "        Overall maintainability - overall, this code is maintainable. \n",
    "        The score scale from -1000000 (strongly agree) to 1000000 (strongly disagree).\n",
    "        You must score in precision, i.e. 14952, -475.456, 757850, 584.58495, 3.141598, etc.\n",
    "        \"\"\"),\n",
    "    ])\n",
    "    chain = prompt | chat.bind_tools(tools=tools) | parser\n",
    "    response = chain.invoke({\n",
    "        \"code\": state[\"code\"],\n",
    "        \"analysis\": state[\"analysis\"]\n",
    "    })\n",
    "    state[\"score\"] = response\n",
    "    return state\n",
    "\n",
    "\n",
    "graph = StateGraph(State)\n",
    "graph.set_entry_point(node_analyze.__name__)\n",
    "graph.add_node(node_analyze.__name__, node_analyze)\n",
    "graph.add_node(node_score.__name__, node_score)\n",
    "graph.add_edge(node_analyze.__name__, node_score.__name__)\n",
    "graph.set_finish_point(node_score.__name__)\n",
    "compiled_graph = graph.compile()\n"
   ],
   "id": "f9375cf75c8a1dd2",
   "outputs": [],
   "execution_count": 40
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:32:56.553322Z",
     "start_time": "2024-07-19T08:32:39.149638Z"
    }
   },
   "cell_type": "code",
   "source": [
    "response = compiled_graph.invoke({\n",
    "    \"code\": dataset.dataframe['code'][0]\n",
    "})"
   ],
   "id": "3f5aa5ba1f30ad1f",
   "outputs": [],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:32:56.556726Z",
     "start_time": "2024-07-19T08:32:56.554198Z"
    }
   },
   "cell_type": "code",
   "source": "response",
   "id": "6acbc8305084cfe3",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'code': 'package def.dom.intl;\\n@jsweet.lang.Interface\\npublic abstract class ResolvedNumberFormatOptions extends def.js.Object {\\n    public String locale;\\n    public String numberingSystem;\\n    public String style;\\n    @jsweet.lang.Optional\\n    public String currency;\\n    @jsweet.lang.Optional\\n    public String currencyDisplay;\\n    public double minimumintegerDigits;\\n    public double minimumFractionDigits;\\n    public double maximumFractionDigits;\\n    @jsweet.lang.Optional\\n    public double minimumSignificantDigits;\\n    @jsweet.lang.Optional\\n    public double maximumSignificantDigits;\\n    public Boolean useGrouping;\\n}\\n\\n',\n",
       " 'analysis': 'To analyze the provided code snippet, we will examine various aspects of readability, understandability, complexity, modularity, and overall maintainability. Here’s a step-by-step analysis:\\n\\n### 1. Readability\\n\\n**Definition**: Readability refers to how easily code can be read and understood by someone who is familiar with the programming language.\\n\\n- **Naming Conventions**: The class name `ResolvedNumberFormatOptions` is descriptive and indicates that it encapsulates options related to number formatting. The properties within the class, such as `locale`, `numberingSystem`, and `style`, are also clear and self-explanatory.\\n- **Whitespace and Formatting**: The code is well formatted with consistent indentation. This enhances readability by visually separating different properties.\\n- **Annotations**: The use of `@jsweet.lang.Optional` annotations indicates which properties are optional. This enhances understanding of the code’s intended use without cluttering it with comments.\\n\\n### 2. Understandability\\n\\n**Definition**: Understandability refers to how easily a programmer can grasp the purpose and logic of the code.\\n\\n- **Abstract Class**: The class is defined as an abstract class which implies that it may be extended. This indicates that it is a base for other classes, although specifics are not provided in this snippet.\\n- **Use of Data Types**: The properties utilize common data types like `String`, `double`, and `Boolean`, which are standard and recognizable. This aids in understanding the types of values that are expected.\\n- **Domain-Specific Language**: The terminology used (e.g., `currency`, `minimumFractionDigits`) suggests a domain-specific context (likely related to internationalization or localization), which may require some background knowledge but is clear for those familiar with it.\\n\\n### 3. Complexity\\n\\n**Definition**: Complexity refers to how complicated the code is, which can affect both its understanding and maintainability.\\n\\n- **Simple Structure**: The class has a straightforward structure with no complex logic or control structures (like loops or conditionals). It primarily serves as a data holder.\\n- **Limited Interactivity**: Since this is a data structure without any methods (besides getters/setters which are not shown), the complexity is minimal. However, complexity may arise in the implementation of the class or the functions that utilize it.\\n\\n### 4. Modularity\\n\\n**Definition**: Modularity describes how well the code is organized into separate components that can be developed, tested, and maintained independently.\\n\\n- **Single Responsibility**: The class appears to follow the single responsibility principle, encapsulating only properties related to number formatting options. This makes it easier to maintain and modify.\\n- **Extensibility**: Being an abstract class, it is designed for extension, allowing other classes to inherit and build upon its properties. This supports modular design.\\n\\n### 5. Overall Maintainability\\n\\n**Definition**: Maintainability assesses how easily the code can be modified to correct faults, improve performance, or adapt to a changed environment.\\n\\n- **Clear Structure**: The clear naming and straightforward structure contribute positively to maintainability. Developers can quickly identify and understand the purpose of each property.\\n- **Documentation**: While the code is somewhat self-explanatory, it lacks comments or documentation to explain the purpose of the class in detail. Adding a brief class-level comment would enhance maintainability by providing context to future developers.\\n- **Potential for Change**: If the requirements for number formatting change, the class can be extended or modified with relative ease due to its modular design.\\n\\n### Conclusion\\n\\nIn summary, the provided code snippet exhibits high readability and understandability due to clear naming and straightforward structure. Its low complexity and modular design contribute positively to maintainability. However, the addition of documentation could further enhance maintainability by offering context to developers unfamiliar with the domain. Overall, the code is well-structured and maintainable within its intended scope.',\n",
       " 'score': [CodeScoring(readability=-900000.0, understandability=-850000.0, complexity=700000.0, modularization=-800000.0, overall_maintainability=-850000.0)]}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 33
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:36:06.520387Z",
     "start_time": "2024-07-19T08:36:06.515718Z"
    }
   },
   "cell_type": "code",
   "source": [
    "x = pd.DataFrame([response[\"score\"][0].dict()])\n",
    "x = ((x + 1000000) / 2000000) * 3 + 1\n",
    "x"
   ],
   "id": "34e7fa367036e7dd",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "   readability  understandability  complexity  modularization  \\\n",
       "0         1.15              1.225        3.55             1.3   \n",
       "\n",
       "   overall_maintainability  \n",
       "0                    1.225  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>readability</th>\n",
       "      <th>understandability</th>\n",
       "      <th>complexity</th>\n",
       "      <th>modularization</th>\n",
       "      <th>overall_maintainability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.15</td>\n",
       "      <td>1.225</td>\n",
       "      <td>3.55</td>\n",
       "      <td>1.3</td>\n",
       "      <td>1.225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 39
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:47:53.238308Z",
     "start_time": "2024-07-19T08:47:53.228857Z"
    }
   },
   "cell_type": "code",
   "source": [
    "selected_dataset: pd.DataFrame = dataset.dataframe.loc[:10]\n",
    "selected_dataset[dataset.dimensions_classes_expectation_value].describe()"
   ],
   "id": "e418da7b7d9e6382",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       readability (class expectation value)  \\\n",
       "count                              11.000000   \n",
       "mean                                1.487410   \n",
       "std                                 0.485618   \n",
       "min                                 1.000000   \n",
       "25%                                 1.058503   \n",
       "50%                                 1.211351   \n",
       "75%                                 1.993192   \n",
       "max                                 2.114131   \n",
       "\n",
       "       understandability (class expectation value)  \\\n",
       "count                                    11.000000   \n",
       "mean                                      1.908151   \n",
       "std                                       0.773503   \n",
       "min                                       1.000008   \n",
       "25%                                       1.186056   \n",
       "50%                                       2.002013   \n",
       "75%                                       2.374359   \n",
       "max                                       3.239380   \n",
       "\n",
       "       complexity (class expectation value)  \\\n",
       "count                             11.000000   \n",
       "mean                               3.438829   \n",
       "std                                0.716716   \n",
       "min                                1.940862   \n",
       "25%                                3.002058   \n",
       "50%                                3.961344   \n",
       "75%                                3.986278   \n",
       "max                                4.000000   \n",
       "\n",
       "       modularization (class expectation value)  \\\n",
       "count                                 11.000000   \n",
       "mean                                   3.558209   \n",
       "std                                    0.512923   \n",
       "min                                    2.833463   \n",
       "25%                                    3.003235   \n",
       "50%                                    3.808855   \n",
       "75%                                    3.970216   \n",
       "max                                    3.986063   \n",
       "\n",
       "       overall (class expectation value)  \n",
       "count                          11.000000  \n",
       "mean                            1.619612  \n",
       "std                             0.639244  \n",
       "min                             1.000000  \n",
       "25%                             1.017510  \n",
       "50%                             1.560691  \n",
       "75%                             2.000013  \n",
       "max                             2.865769  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>readability (class expectation value)</th>\n",
       "      <th>understandability (class expectation value)</th>\n",
       "      <th>complexity (class expectation value)</th>\n",
       "      <th>modularization (class expectation value)</th>\n",
       "      <th>overall (class expectation value)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.487410</td>\n",
       "      <td>1.908151</td>\n",
       "      <td>3.438829</td>\n",
       "      <td>3.558209</td>\n",
       "      <td>1.619612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.485618</td>\n",
       "      <td>0.773503</td>\n",
       "      <td>0.716716</td>\n",
       "      <td>0.512923</td>\n",
       "      <td>0.639244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000008</td>\n",
       "      <td>1.940862</td>\n",
       "      <td>2.833463</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.058503</td>\n",
       "      <td>1.186056</td>\n",
       "      <td>3.002058</td>\n",
       "      <td>3.003235</td>\n",
       "      <td>1.017510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.211351</td>\n",
       "      <td>2.002013</td>\n",
       "      <td>3.961344</td>\n",
       "      <td>3.808855</td>\n",
       "      <td>1.560691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.993192</td>\n",
       "      <td>2.374359</td>\n",
       "      <td>3.986278</td>\n",
       "      <td>3.970216</td>\n",
       "      <td>2.000013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.114131</td>\n",
       "      <td>3.239380</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.986063</td>\n",
       "      <td>2.865769</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 47
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:39:42.591973Z",
     "start_time": "2024-07-19T08:39:19.081840Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from concurrent import futures\n",
    "\n",
    "with futures.ThreadPoolExecutor() as executor:\n",
    "    tasks = []\n",
    "    for index, row in selected_dataset.iterrows():\n",
    "        kwargs = {\n",
    "            \"input\": {\"code\": row[\"code\"]}\n",
    "        }\n",
    "        task = executor.submit(compiled_graph.invoke, **kwargs)\n",
    "        tasks.append(task)\n",
    "    responses = [task.result() for task in tasks]"
   ],
   "id": "564d44efd009186a",
   "outputs": [],
   "execution_count": 42
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T09:04:34.012519Z",
     "start_time": "2024-07-19T09:04:34.003240Z"
    }
   },
   "cell_type": "code",
   "source": [
    "predicted = pd.DataFrame([response[\"score\"][0].dict() for response in responses])\n",
    "predicted = predicted.rename(columns={\"overall_maintainability\": \"overall\"})\n",
    "predicted = ((predicted + 1000000) / 2000000) * 3 + 1\n",
    "predicted.describe()"
   ],
   "id": "3c20f6ca8917bc48",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       readability  understandability  complexity  modularization    overall\n",
       "count    11.000000          11.000000   11.000000       11.000000  11.000000\n",
       "mean      1.859091           1.872727    3.052273        2.670455   1.985227\n",
       "std       0.364910           0.356976    0.207501        0.515366   0.312891\n",
       "min       1.225000           1.225000    2.725000        1.450000   1.375000\n",
       "25%       1.637500           1.675000    2.950000        2.575000   1.787500\n",
       "50%       2.050000           2.050000    3.025000        2.875000   2.125000\n",
       "75%       2.125000           2.087500    3.137500        2.950000   2.200000\n",
       "max       2.200000           2.275000    3.550000        3.100000   2.275000"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>readability</th>\n",
       "      <th>understandability</th>\n",
       "      <th>complexity</th>\n",
       "      <th>modularization</th>\n",
       "      <th>overall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.859091</td>\n",
       "      <td>1.872727</td>\n",
       "      <td>3.052273</td>\n",
       "      <td>2.670455</td>\n",
       "      <td>1.985227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.364910</td>\n",
       "      <td>0.356976</td>\n",
       "      <td>0.207501</td>\n",
       "      <td>0.515366</td>\n",
       "      <td>0.312891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.225000</td>\n",
       "      <td>1.225000</td>\n",
       "      <td>2.725000</td>\n",
       "      <td>1.450000</td>\n",
       "      <td>1.375000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.637500</td>\n",
       "      <td>1.675000</td>\n",
       "      <td>2.950000</td>\n",
       "      <td>2.575000</td>\n",
       "      <td>1.787500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.050000</td>\n",
       "      <td>2.050000</td>\n",
       "      <td>3.025000</td>\n",
       "      <td>2.875000</td>\n",
       "      <td>2.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.125000</td>\n",
       "      <td>2.087500</td>\n",
       "      <td>3.137500</td>\n",
       "      <td>2.950000</td>\n",
       "      <td>2.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.200000</td>\n",
       "      <td>2.275000</td>\n",
       "      <td>3.550000</td>\n",
       "      <td>3.100000</td>\n",
       "      <td>2.275000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 51
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:40:02.087812Z",
     "start_time": "2024-07-19T08:40:02.082210Z"
    }
   },
   "cell_type": "code",
   "source": [
    "expected = selected_dataset[dataset.dimensions_classes_expectation_value].rename(\n",
    "    columns=dict(zip(dataset.dimensions_classes_expectation_value, dataset.dimensions)))\n",
    "expected"
   ],
   "id": "ebceba65e36fa357",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "    readability  understandability  complexity  modularization   overall\n",
       "0      1.028807           1.111403    3.975756        3.979663  1.013828\n",
       "1      1.000000           1.000008    4.000000        3.975870  1.000001\n",
       "2      1.986384           2.740079    2.591814        2.833463  2.276708\n",
       "3      1.211351           1.697925    3.961344        2.838980  1.927840\n",
       "4      2.114131           3.239380    1.940862        3.004811  2.865769\n",
       "5      2.000000           2.844343    3.000000        3.001660  2.000000\n",
       "6      1.088199           1.260710    3.984427        3.964561  1.149678\n",
       "7      1.830838           2.008639    3.004115        3.808855  1.560691\n",
       "8      1.099956           1.077158    3.988128        3.946899  1.021191\n",
       "9      2.001849           2.008008    3.380674        3.986063  2.000025\n",
       "10     1.000000           2.002013    4.000000        3.799478  1.000000"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>readability</th>\n",
       "      <th>understandability</th>\n",
       "      <th>complexity</th>\n",
       "      <th>modularization</th>\n",
       "      <th>overall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.028807</td>\n",
       "      <td>1.111403</td>\n",
       "      <td>3.975756</td>\n",
       "      <td>3.979663</td>\n",
       "      <td>1.013828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000008</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.975870</td>\n",
       "      <td>1.000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.986384</td>\n",
       "      <td>2.740079</td>\n",
       "      <td>2.591814</td>\n",
       "      <td>2.833463</td>\n",
       "      <td>2.276708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.211351</td>\n",
       "      <td>1.697925</td>\n",
       "      <td>3.961344</td>\n",
       "      <td>2.838980</td>\n",
       "      <td>1.927840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.114131</td>\n",
       "      <td>3.239380</td>\n",
       "      <td>1.940862</td>\n",
       "      <td>3.004811</td>\n",
       "      <td>2.865769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.844343</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.001660</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.088199</td>\n",
       "      <td>1.260710</td>\n",
       "      <td>3.984427</td>\n",
       "      <td>3.964561</td>\n",
       "      <td>1.149678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.830838</td>\n",
       "      <td>2.008639</td>\n",
       "      <td>3.004115</td>\n",
       "      <td>3.808855</td>\n",
       "      <td>1.560691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.099956</td>\n",
       "      <td>1.077158</td>\n",
       "      <td>3.988128</td>\n",
       "      <td>3.946899</td>\n",
       "      <td>1.021191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.001849</td>\n",
       "      <td>2.008008</td>\n",
       "      <td>3.380674</td>\n",
       "      <td>3.986063</td>\n",
       "      <td>2.000025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.002013</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.799478</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 44
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T09:02:06.171598Z",
     "start_time": "2024-07-19T09:02:06.158967Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from rexmex import RatingMetricSet, ScoreCard, ClassificationMetricSet\n",
    "\n",
    "\n",
    "def evaluate_classification(y_true, y_pred):\n",
    "    metric_set = ClassificationMetricSet()\n",
    "    score_card = ScoreCard(metric_set)\n",
    "    report = score_card.get_performance_metrics(y_true, y_pred)\n",
    "    print(report)\n",
    "\n",
    "\n",
    "def evaluate_regression(y_true, y_pred):\n",
    "    metric_set = RatingMetricSet()\n",
    "    score_card = ScoreCard(metric_set)\n",
    "    report = score_card.get_performance_metrics(y_true, y_pred)\n",
    "    print(report)\n",
    "\n",
    "\n",
    "\n"
   ],
   "id": "8aeba679b18852ac",
   "outputs": [],
   "execution_count": 49
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "for dimension in dataset.dimensions:\n",
    "    print(f'Evaluation for dimension: {dimension}')\n",
    "    evaluate_classification(expected[dimension], predicted[dimension])"
   ],
   "id": "7dbd9533c196b92f"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T09:50:23.091360Z",
     "start_time": "2024-07-19T09:50:23.073995Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for dimension in dataset.dimensions:\n",
    "    print(f'Evaluation for dimension: {dimension}')\n",
    "    evaluate_regression(expected[dimension], predicted[dimension])"
   ],
   "id": "c13948b82aa24750",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation for dimension: readability\n",
      "       mae       mse      rmse     mape      smape  r_squared  \\\n",
      "0  0.62211  0.566435  0.752619  0.52289  39.199617  -1.642127   \n",
      "\n",
      "   pearson_correlation  \n",
      "0            -0.288176  \n",
      "Evaluation for dimension: understandability\n",
      "        mae       mse      rmse      mape      smape  r_squared  \\\n",
      "0  0.829816  0.795043  0.891652  0.511766  45.199515  -0.461704   \n",
      "\n",
      "   pearson_correlation  \n",
      "0            -0.266962  \n",
      "Evaluation for dimension: complexity\n",
      "        mae       mse      rmse      mape      smape  r_squared  \\\n",
      "0  0.680615  0.586261  0.765677  0.205652  21.071402  -0.255423   \n",
      "\n",
      "   pearson_correlation  \n",
      "0             0.256251  \n",
      "Evaluation for dimension: modularization\n",
      "        mae      mse      rmse      mape      smape  r_squared  \\\n",
      "0  0.908943  1.26288  1.123779  0.241379  29.731535  -4.280195   \n",
      "\n",
      "   pearson_correlation  \n",
      "0             0.012188  \n",
      "Evaluation for dimension: overall\n",
      "        mae      mse     rmse      mape      smape  r_squared  \\\n",
      "0  0.682434  0.61251  0.78263  0.542941  40.053668  -0.648818   \n",
      "\n",
      "   pearson_correlation  \n",
      "0            -0.050461  \n"
     ]
    }
   ],
   "execution_count": 56
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T09:58:40.510580Z",
     "start_time": "2024-07-19T09:58:40.457466Z"
    }
   },
   "cell_type": "code",
   "source": [
    "plot = pd.DataFrame({\n",
    "    \"expected\": [i for i in range(10)],\n",
    "    \"predicted\": [i for i in range(10)]\n",
    "})\n",
    "plot.plot.scatter(x=\"expected\", y=\"predicted\")"
   ],
   "id": "f0f130082b3b6862",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='expected', ylabel='predicted'>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAGwCAYAAACHJU4LAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnj0lEQVR4nO3df3RU9Z3/8dckJJMfkClkkigSSDCp/FQDaRBi06VGWbXtck4rbQ5dAbF13UiEbD2AXXQPCpE9laWGFosKLYsGPVVb/LmyrJASQSAQKMuPJCYBqgITAhmStAlM7vcPv8wyBmIyJLmfkOfjnDmHvO+dyUtGnde593PvOCzLsgQAAGCgELsDAAAAXAlFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWP3sDnA1Wltb9dlnn2nAgAFyOBx2xwEAAB1gWZbOnTunwYMHKySk/WMmvbqofPbZZ0pMTLQ7BgAACMLx48c1ZMiQdvfp1UVlwIABkr74B42JibE5DQAA6Aiv16vExET/53h7enVRuXi6JyYmhqICAEAv05FlGyymBQAAxqKoAAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABj9epb6AMAgO5T5WnQ0bomJcVGK9kdbUsGigoAAAhwtqlFeUVlKq7w+GdZqXEqzEmTKyqsR7Nw6gcAAATIKypTSWVtwKykslZzivb2eBaKCgAA8KvyNKi4wiOfZQXMfZal4gqPqmsbezQPRQUAAPgdrWtqd3vNaYoKAACwybBBUe1uT4rt2UW1FBUAAOA3PK6/slLjFOpwBMxDHQ5lpcb1+NU/FBUAABCgMCdNmSnugFlmiluFOWk9noXLkwEAQABXVJjWzc5QdW2jak43ch8VAABgnmS3fQXlIk79AAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABjUVQAAICxKCoAAMBYFBUAAGAsigoAADAWRQUAABiLogIAAIxFUQEAAMaiqAAAAGNRVAAAgLEoKgAAwFgUFQAAYCyKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWP3sDgAAwLWmytOgo3VNSoqNVrI72u44vZqtRcXn8+nf/u3ftH79ep04cUKDBw/WzJkz9a//+q9yOBx2RgMAoNPONrUor6hMxRUe/ywrNU6FOWlyRYXZmKz3srWoLFu2TKtWrdLvfvc7jR49Wrt379asWbPkcrmUl5dnZzQAADotr6hMJZW1AbOSylrNKdqrdbMzbErVu9laVD766CP9wz/8g+69915JUlJSkoqKirRz587L7t/c3Kzm5mb/z16vt0dyAgDwVao8DQFHUi7yWZaKKzyqrm3kNFAQbF1MO2nSJG3evFnl5eWSpH379mnbtm26++67L7t/QUGBXC6X/5GYmNiTcQEAuKKjdU3tbq853dhDSa4tth5RWbBggbxer0aMGKHQ0FD5fD4tWbJE06dPv+z+CxcuVH5+vv9nr9dLWQEAGGHYoKh2tyfFcjQlGLYWlddee00vv/yyXnnlFY0ePVplZWWaO3euBg8erBkzZrTZ3+l0yul02pAUAID2DY/rr6zUOJVU1spnWf55qMOhzBQ3p32C5LCsS/42e1hiYqIWLFig3Nxc/+zpp5/W+vXrdfjw4a98vtfrlcvlUn19vWJiYrozKgAAX6m+6bzmFO3lqp+v0JnPb1uPqDQ1NSkkJHCZTGhoqFpbW21KBABA8FxRYVo3O0PVtY2qOd3IfVS6gK1F5bvf/a6WLFmioUOHavTo0dq7d6+WL1+uBx54wM5YAABclWQ3BaWr2Hrq59y5c1q0aJHefPNNnTp1SoMHD1ZOTo6eeOIJhYeHf+XzOfUDAEDv05nPb1uLytWiqAAA0Pt05vObLyUEAADGoqgAAABjUVQAAICxKCoAAMBYFBUAAGAsigoAADAWRQUAABiLogIAAIxFUQEAAMaiqAAAAGNRVAAAgLEoKgAAwFgUFQAAYCyKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABj9bM7AAAAklTladDRuiYlxUYr2R1tdxwYgqICALDV2aYW5RWVqbjC459lpcapMCdNrqgwG5PBBJz6AQDYKq+oTCWVtQGzkspazSnaa1MimISiAgCwTZWnQcUVHvksK2DusywVV3hUXdtoUzKYgqICALDN0bqmdrfXnKao9HUUFQCAbYYNimp3e1Isi2r7OooKAMA2w+P6Kys1TqEOR8A81OFQVmocV/+AogIAsFdhTpoyU9wBs8wUtwpz0mxKBJNweTIAwFauqDCtm52h6tpG1Zxu5D4qCEBRAQAYIdlNQUFbnPoBAADGoqgAAABjUVQAAICxKCoAAMBYFBUAAGAsigoAADAWRQUAABiLogIAAIxFUQEAAMaiqAAAAGNRVAAAgLEoKgAAwFgUFQAAYCyKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABjUVQAAICxbC8qn376qX784x8rNjZWkZGRGjt2rHbv3m13LADoNao8DfrwyClV1zbaHQXocv3s/OVnzpxRZmamJk+erPfee09xcXGqqKjQwIED7YwFAL3C2aYW5RWVqbjC459lpcapMCdNrqgwG5MBXcfWorJs2TIlJiZq7dq1/llycrKNiQCg98grKlNJZW3ArKSyVnOK9mrd7AybUgFdy9ZTPxs3blR6erruu+8+xcfHKy0tTS+88MIV929ubpbX6w14AEBfVOVpUHGFRz7LCpj7LEvFFR5OA+GaYWtRqaqq0qpVq5Samqr/+q//0sMPP6y8vDz97ne/u+z+BQUFcrlc/kdiYmIPJwYAMxyta2p3e81pigquDQ7L+lId70Hh4eFKT0/XRx995J/l5eVp165d2r59e5v9m5ub1dzc7P/Z6/UqMTFR9fX1iomJ6ZHMAGCCKk+Dvv3s1itu//Bnf6dkd3QPJgI6zuv1yuVydejz29YjKtdff71GjRoVMBs5cqSOHTt22f2dTqdiYmICHgDQFw2P66+s1DiFOhwB81CHQ1mpcZQUXDNsLSqZmZk6cuRIwKy8vFzDhg2zKREA9B6FOWnKTHEHzDJT3CrMSbMpEdD1bL3qZ968eZo0aZKWLl2qadOmaefOnVq9erVWr15tZywA6BVcUWFaNztD1bWNqjndqKTYaI6k4Jpj6xoVSXr77be1cOFCVVRUKDk5Wfn5+frJT37Soed25hwXAAAwQ2c+v20vKleDogIAQO/TaxbTAgAAtIeiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABjUVQAAICxKCoAAMBYFBUAAGAsigoAADAWRQUAABiLogIAAIxFUQEAAMaiqAAAAGNRVAAAgLH6dXTHtLQ0ORyODu27Z8+eoAMBAABc1OGiMnXqVP+f//a3v+nXv/61Ro0apYkTJ0qSduzYof/93//VP//zP3d5SAAA0Dd1uKg8+eST/j8/+OCDysvL01NPPdVmn+PHj3ddOgAA0Kc5LMuyOvskl8ul3bt3KzU1NWBeUVGh9PR01dfXd1nA9ni9XrlcLtXX1ysmJqZHficAALg6nfn8DmoxbWRkpEpKStrMS0pKFBEREcxLAgAAtNHhUz+Xmjt3rh5++GHt2bNHGRkZkqSPP/5Ya9as0aJFi7o0IAB0lypPg47WNSkpNlrJ7mi74wC4jKCKyoIFCzR8+HD98pe/1Pr16yVJI0eO1Nq1azVt2rQuDQgAXe1sU4vyispUXOHxz7JS41SYkyZXVJiNyQB8WVBrVEzBGhUAwbj/pZ0qqayV75L//YU6HMpMcWvd7AwbkwF9Q7evUZGks2fP6sUXX9Tjjz+uuro6SV/cP+XTTz8N9iUBoNtVeRpUXOEJKCmS5LMsFVd4VF3baFMyAJcT1Kmf/fv3Kzs7Wy6XSzU1NXrwwQc1aNAgvfHGGzp27JjWrVvX1TkBoEscrWtqd3vN6UbWqwAGCeqISn5+vmbOnKmKioqAq3zuueceFRcXd1k4AOhqwwZFtbs9KZaSApgkqKKya9cuPfTQQ23mN9xwg06cOHHVoQCguwyP66+s1DiFfukrQUIdDmWlxnE0BTBMUEXF6XTK6/W2mZeXlysuLu6qQwFAdyrMSVNmijtglpniVmFOmk2JAFxJUGtUvve972nx4sV67bXXJEkOh0PHjh3T/Pnz9f3vf79LAwJAV3NFhWnd7AxV1zaq5nQj91EBDBbU5cn19fX6wQ9+oN27d+vcuXMaPHiwTpw4oYkTJ+rdd99VdHTP/AfP5ckAAPQ+nfn8DuqIisvl0qZNm1RSUqJ9+/apoaFB48aNU3Z2dlCBAQAALieoorJu3Tr98Ic/VGZmpjIzM/3zlpYWbdiwQffff3+XBQQAAH1XUKd+QkND9fnnnys+Pj5gfvr0acXHx8vn83VZwPZw6gcAgN6n2+9Ma1mWHF+6tE+S/vKXv8jlcgXzkgAAAG106tRPWlqaHA6HHA6H7rjjDvXr939P9/l8qq6u1t///d93eUgAANA3daqoTJ06VZJUVlamKVOmqH///v5t4eHhSkpK4vJkAADQZTpVVJ588klJUlJSkn70ox/J6XR2SygAAAApyDUqo0aNUllZWZv5xx9/rN27d19tJgAAAElBFpXc3FwdP368zfzTTz9Vbm7uVYcCAACQgiwqBw8e1Lhx49rM09LSdPDgwasOBQAAIF3FlxKePHmyzfzzzz8PuBIIAADgagRVVO666y4tXLhQ9fX1/tnZs2f1+OOP68477+yycAAAoG8L6vDHL37xC2VlZWnYsGFKS/via9HLysqUkJCg//zP/+zSgAAAoO8KqqjccMMN2r9/v15++WXt27dPkZGRmjVrlnJychQWFtbVGQEAQB8V9IKS6Oho/fSnP+3KLAAAAAE6XFQ2btyou+++W2FhYdq4cWO7+37ve9+76mAAAAAd/vbkkJAQnThxQvHx8QoJufIaXIfDwbcnAwCAK+rM53eHj6i0trZe9s8AAADdJajLkwEAAHpCh4+oPPfccx1+0by8vKDCAAAAXKrDa1SSk5MDfvZ4PGpqatLXvvY1SV/c8C0qKkrx8fGqqqrq8qCXwxoVAAB6n858fnf41E91dbX/sWTJEt166606dOiQ6urqVFdXp0OHDmncuHF66qmnrvofAAAAQOrEEZVL3Xjjjfr973/vvyvtRaWlpfrBD36g6urqLgvYHo6oAADQ+3TLEZVLff7557pw4UKbuc/nu+yXFQIAAAQjqKJyxx136KGHHtKePXv8s9LSUj388MPKzs7usnAAAKBvC6qorFmzRtddd53S09PldDrldDqVkZGhhIQEvfjii12dEQAA9FFBfddPXFyc3n33XZWXl+vw4cOSpBEjRujrX/96l4YDAAB9W9BfSihJSUlJsixLN954o/r1u6qXAgAAaCOoUz9NTU2aPXu2oqKiNHr0aB07dkySNGfOHD3zzDNBBXnmmWfkcDg0d+7coJ4PoOdUeRr04ZFTqq5ttDsKgGtcUEVl4cKF2rdvn7Zs2aKIiAj/PDs7W6+++mqnX2/Xrl36zW9+o5tvvjmYOAB6yNmmFt3/0k59+9mtmrV2lyb/Yovuf2mn6pvO2x0NwDUqqKLyhz/8QStXrtTtt98uh8Phn48ePVqffPJJp16roaFB06dP1wsvvKCBAwcGEwdAD8krKlNJZW3ArKSyVnOK9tqUCMC1Lqii4vF4FB8f32be2NgYUFw6Ijc3V/fee2+HLmtubm6W1+sNeADoGVWeBhVXeOT70j0ifZal4goPp4EAdIugikp6erreeecd/88Xy8mLL76oiRMndvh1NmzYoD179qigoKBD+xcUFMjlcvkfiYmJnQsOIGhH65ra3V5zmqICoOsFdanO0qVLdffdd+vgwYO6cOGCfvnLX+rgwYP66KOPtHXr1g69xvHjx/Xoo49q06ZNAetc2rNw4ULl5+f7f/Z6vZQVoIcMGxTV7vak2OgeSgKgLwnqiMrtt9+uffv26cKFCxo7dqw++OADxcfHa/v27Ro/fnyHXqO0tFSnTp3SuHHj1K9fP/Xr109bt27Vc889p379+snn87V5jtPpVExMTMADQM8YHtdfWalxCv3S6d1Qh0NZqXFKdlNUAHS9Tn8p4fnz5/XQQw9p0aJFSk5ODvoXnzt3TkePHg2YzZo1SyNGjND8+fM1ZsyYr3wNvpQQ6Fn1Tec1p2iviis8/llWapwKc9LkigqzMRmA3qQzn9+dPvUTFham119/XYsWLQo6oCQNGDCgTRmJjo5WbGxsh0oKgJ7nigrTutkZqq5tVM3pRiXFRnMkBUC3CurUz9SpU/WHP/yhi6MA6C2S3dGafFM8JQVAtwtqMW1qaqoWL16skpISjR8/XtHRgf+zysvLCyrMli1bgnoeAAC4NnV6jYqkdtemOBwOVVVVXVWojmKNCgAAvU+3rlGRpOrqav+fL/aczt7oDQAA4KsEtUZFkl566SWNGTNGERERioiI0JgxY/Tiiy92ZTYAANDHBXVE5YknntDy5cs1Z84c/51ot2/frnnz5unYsWNavHhxl4YEAAB9U1BrVOLi4vTcc88pJycnYF5UVKQ5c+aotrb2Cs/sWqxRAQCg9+nM53dQp37Onz+v9PT0NvPx48frwoULwbwkAABAG0EVlX/8x3/UqlWr2sxXr16t6dOnX3UoAAAAKcg1KtIXi2k/+OAD3XbbbZKkjz/+WMeOHdP9998f8MWBy5cvv/qUAACgTwqqqBw4cEDjxo2TJH3yySeSJLfbLbfbrQMHDvj345JlAABwNYIqKh9++GFX5wAAAGgj6PuoAAAAdDeKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABjUVQAAICxKCoAAMBY/ewOAPQlVZ4GHa1rUlJstJLd0XbHAQDjUVSAHnC2qUV5RWUqrvD4Z1mpcSrMSZMrKszGZABgNk79AD0gr6hMJZW1AbOSylrNKdprUyIA6B0oKkA3q/I0qLjCI59lBcx9lqXiCo+qaxttSgYA5qOoAN3saF1Tu9trTlNUAOBKKCpANxs2KKrd7UmxLKoFgCuhqADdbHhcf2WlxinU4QiYhzocykqN4+ofAGgHRQXoAYU5acpMcQfMMlPcKsxJsykRAPQOXJ4M9ABXVJjWzc5QdW2jak43ch8VAOggigrQg5LdFBQA6AxO/QAAAGNRVAAAgLEoKgAAwFgUFQAAYCyKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABjUVQAAICxKCoAAMBYFBUAAGAsigoAADAWRQUAABiLogIAAIxFUQEAAMaiqAAAAGNRVAAAgLEoKgAAwFj97A4AdESVp0FH65qUFButZHe03XEAAD3E1qJSUFCgN954Q4cPH1ZkZKQmTZqkZcuW6aabbrIzFgxytqlFeUVlKq7w+GdZqXEqzEmTKyrMxmQAgJ5g66mfrVu3Kjc3Vzt27NCmTZt0/vx53XXXXWpsbLQzFgySV1SmksragFlJZa3mFO21KREAoCfZekTl/fffD/j5t7/9reLj41VaWqqsrKw2+zc3N6u5udn/s9fr7faMsE+VpyHgSMpFPstScYVH1bWNnAYCgGucUYtp6+vrJUmDBg267PaCggK5XC7/IzExsSfjoYcdrWtqd3vNaY68AcC1zpii0traqrlz5yozM1Njxoy57D4LFy5UfX29/3H8+PEeTomeNGxQVLvbk2I5mgIA1zpjrvrJzc3VgQMHtG3btivu43Q65XQ6ezAV7DQ8rr+yUuNUUlkrn2X556EOhzJT3Jz2AYA+wIgjKo888ojefvttffjhhxoyZIjdcWCQwpw0Zaa4A2aZKW4V5qTZlAgA0JNsPaJiWZbmzJmjN998U1u2bFFycrKdcWAgV1SY1s3OUHVto2pON3IfFQDoY2wtKrm5uXrllVf0xz/+UQMGDNCJEyckSS6XS5GRkXZGg2GS3RQUAOiLHJZ1ycn/nv7lDsdl52vXrtXMmTO/8vler1cul0v19fWKiYnp4nQAAKA7dObz2/ZTPwAAAFdixGJaAACAy6GoAAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABjUVQAAICxKCoAAMBYFBUAAGAsigoAADAWRQUAABiLogIAAIxFUQEAAMaiqAAAAGNRVAAAgLEoKgAAwFgUFQAAYCyKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY/WzOwC6V5WnQUfrmpQUG61kd7TdcQAA6BSKyjXqbFOL8orKVFzh8c+yUuNUmJMmV1SYjckAAOg4Tv1co/KKylRSWRswK6ms1ZyivTYlAgCg8ygq16AqT4OKKzzyWVbA3GdZKq7wqLq20aZkAAB0DkXlGnS0rqnd7TWnKSoAgN6BonINGjYoqt3tSbEsqgUA9A4UlWvQ8Lj+ykqNU6jDETAPdTiUlRrH1T8AgF6DonKNKsxJU2aKO2CWmeJWYU6aTYkAAOg8Lk++RrmiwrRudoaqaxtVc7qR+6gAAHoliso1LtlNQQEA9F6c+gEAAMaiqAAAAGNRVAAAgLEoKgAAwFgUFQAAYCyKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWBQVAABgLIoKAAAwFkUFAAAYi6ICAACMRVEBAADGoqgAAABjUVQAAICxKCoAAMBYFBUAAGAsigoAADAWRQUAABiLogIAAIxFUQEAAMaiqAAAAGNRVAAAgLGMKCq/+tWvlJSUpIiICE2YMEE7d+60O5KqPA368MgpVdc22h0FAIA+q5/dAV599VXl5+fr+eef14QJE7RixQpNmTJFR44cUXx8fI/nOdvUoryiMhVXePyzrNQ4FeakyRUV1uN5AADoy2w/orJ8+XL95Cc/0axZszRq1Cg9//zzioqK0po1a2zJk1dUppLK2oBZSWWt5hTttSUPAAB9ma1FpaWlRaWlpcrOzvbPQkJClJ2dre3bt7fZv7m5WV6vN+DRlao8DSqu8MhnWQFzn2WpuMLDaSAAAHqYrUWltrZWPp9PCQkJAfOEhASdOHGizf4FBQVyuVz+R2JiYpfmOVrX1O72mtMUFQAAepLtp346Y+HChaqvr/c/jh8/3qWvP2xQVLvbk2Kju/T3AQCA9tm6mNbtdis0NFQnT54MmJ88eVLXXXddm/2dTqecTme35Rke119ZqXEqqawNOP0T6nAoM8WtZDdFBQCAnmTrEZXw8HCNHz9emzdv9s9aW1u1efNmTZw40ZZMhTlpykxxB8wyU9wqzEmzJQ8AAH2Z7Zcn5+fna8aMGUpPT1dGRoZWrFihxsZGzZo1y5Y8rqgwrZudoeraRtWcblRSbDRHUgAAsIntReWHP/yhPB6PnnjiCZ04cUK33nqr3n///TYLbHtaspuCAgCA3RyW9aVrcXsRr9crl8ul+vp6xcTE2B0HAAB0QGc+v3vVVT8AAKBvoagAAABjUVQAAICxKCoAAMBYFBUAAGAsigoAADAWRQUAABiLogIAAIxFUQEAAMay/Rb6V+PiTXW9Xq/NSQAAQEdd/NzuyM3xe3VROXfunCQpMTHR5iQAAKCzzp07J5fL1e4+vfq7flpbW/XZZ59pwIABcjgcXfraXq9XiYmJOn78ON8jZADeD7PwfpiF98M8vCftsyxL586d0+DBgxUS0v4qlF59RCUkJERDhgzp1t8RExPDv2QG4f0wC++HWXg/zMN7cmVfdSTlIhbTAgAAY1FUAACAsSgqV+B0OvXkk0/K6XTaHQXi/TAN74dZeD/Mw3vSdXr1YloAAHBt44gKAAAwFkUFAAAYi6ICAACMRVEBAADGoqhcxq9+9SslJSUpIiJCEyZM0M6dO+2O1GcVFBToG9/4hgYMGKD4+HhNnTpVR44csTsWJD3zzDNyOByaO3eu3VH6tE8//VQ//vGPFRsbq8jISI0dO1a7d++2O1af5PP5tGjRIiUnJysyMlI33nijnnrqqQ59nw2ujKLyJa+++qry8/P15JNPas+ePbrllls0ZcoUnTp1yu5ofdLWrVuVm5urHTt2aNOmTTp//rzuuusuNTY22h2tT9u1a5d+85vf6Oabb7Y7Sp925swZZWZmKiwsTO+9954OHjyoZ599VgMHDrQ7Wp+0bNkyrVq1SitXrtShQ4e0bNky/fu//7sKCwvtjtarcXnyl0yYMEHf+MY3tHLlSklffJ9QYmKi5syZowULFticDh6PR/Hx8dq6dauysrLsjtMnNTQ0aNy4cfr1r3+tp59+WrfeeqtWrFhhd6w+acGCBSopKdGf/vQnu6NA0ne+8x0lJCTopZde8s++//3vKzIyUuvXr7cxWe/GEZVLtLS0qLS0VNnZ2f5ZSEiIsrOztX37dhuT4aL6+npJ0qBBg2xO0nfl5ubq3nvvDfjvBPbYuHGj0tPTdd999yk+Pl5paWl64YUX7I7VZ02aNEmbN29WeXm5JGnfvn3atm2b7r77bpuT9W69+ksJu1ptba18Pp8SEhIC5gkJCTp8+LBNqXBRa2ur5s6dq8zMTI0ZM8buOH3Shg0btGfPHu3atcvuKJBUVVWlVatWKT8/X48//rh27dqlvLw8hYeHa8aMGXbH63MWLFggr9erESNGKDQ0VD6fT0uWLNH06dPtjtarUVTQa+Tm5urAgQPatm2b3VH6pOPHj+vRRx/Vpk2bFBERYXcc6Ivynp6erqVLl0qS0tLSdODAAT3//PMUFRu89tprevnll/XKK69o9OjRKisr09y5czV48GDej6tAUbmE2+1WaGioTp48GTA/efKkrrvuOptSQZIeeeQRvf322youLtaQIUPsjtMnlZaW6tSpUxo3bpx/5vP5VFxcrJUrV6q5uVmhoaE2Jux7rr/+eo0aNSpgNnLkSL3++us2JerbHnvsMS1YsEA/+tGPJEljx47V0aNHVVBQQFG5CqxRuUR4eLjGjx+vzZs3+2etra3avHmzJk6caGOyvsuyLD3yyCN688039T//8z9KTk62O1Kfdccdd+jPf/6zysrK/I/09HRNnz5dZWVllBQbZGZmtrlcv7y8XMOGDbMpUd/W1NSkkJDAj9XQ0FC1trbalOjawBGVL8nPz9eMGTOUnp6ujIwMrVixQo2NjZo1a5bd0fqk3NxcvfLKK/rjH/+oAQMG6MSJE5Ikl8ulyMhIm9P1LQMGDGizNig6OlqxsbGsGbLJvHnzNGnSJC1dulTTpk3Tzp07tXr1aq1evdruaH3Sd7/7XS1ZskRDhw7V6NGjtXfvXi1fvlwPPPCA3dF6NwttFBYWWkOHDrXCw8OtjIwMa8eOHXZH6rMkXfaxdu1au6PBsqxvfetb1qOPPmp3jD7trbfessaMGWM5nU5rxIgR1urVq+2O1Gd5vV7r0UcftYYOHWpFRERYw4cPt37+859bzc3Ndkfr1biPCgAAMBZrVAAAgLEoKgAAwFgUFQAAYCyKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAOASW7ZskcPh0NmzZ+2OAkAUFQDXAMoFcO2iqAAAAGNRVAB8pdbWVhUUFCg5OVmRkZG65ZZb9Pvf/16WZSk7O1tTpkzRxa8Nq6ur05AhQ/TEE09I+r+jHe+8845uvvlmRURE6LbbbtOBAwcCfse2bdv0zW9+U5GRkUpMTFReXp4aGxv925ubmzV//nwlJibK6XQqJSVFL730kmpqajR58mRJ0sCBA+VwODRz5sx2c1/q3Xff1de//nVFRkZq8uTJqqmp6aa/RQBBsfc7EQH0Bk8//bQ1YsQI6/3337c++eQTa+3atZbT6bS2bNli/eUvf7EGDhxorVixwrIsy7rvvvusjIwM6/z585ZlWdaHH35oSbJGjhxpffDBB9b+/fut73znO1ZSUpLV0tJiWZZlVVZWWtHR0dZ//Md/WOXl5VZJSYmVlpZmzZw5059h2rRpVmJiovXGG29Yn3zyifXf//3f1oYNG6wLFy5Yr7/+uiXJOnLkiPX5559bZ8+e/crclmVZx44ds5xOp5Wfn28dPnzYWr9+vZWQkGBJss6cOdODf8MAroSiAqBdf/vb36yoqCjro48+CpjPnj3bysnJsSzLsl577TUrIiLCWrBggRUdHW2Vl5f797tYVDZs2OCfnT592oqMjLReffVV/2v99Kc/DXj9P/3pT1ZISIj117/+1Tpy5Iglydq0adNlM178HZeWi47kXrhwoTVq1KiA7fPnz6eoAAbpZ+PBHAC9QGVlpZqamnTnnXcGzFtaWpSWliZJuu+++/Tmm2/qmWee0apVq5SamtrmdSZOnOj/86BBg3TTTTfp0KFDkqR9+/Zp//79evnll/37WJal1tZWVVdX689//rNCQ0P1rW99q0tzHzp0SBMmTLhiTgD2o6gAaFdDQ4Mk6Z133tENN9wQsM3pdEqSmpqaVFpaqtDQUFVUVAT1Ox566CHl5eW12TZ06FBVVlZ2S24A5qOoAGjXqFGj5HQ6dezYsSse0fiXf/kXhYSE6L333tM999yje++9V9/+9rcD9tmxY4eGDh0qSTpz5ozKy8s1cuRISdK4ceN08OBBpaSkXPb1x44dq9bWVm3dulXZ2dlttoeHh0uSfD5fp3KPHDlSGzdubJMTgDkoKgDaNWDAAP3sZz/TvHnz1Nraqttvv1319fUqKSlRTEyM3G631qxZo+3bt2vcuHF67LHHNGPGDO3fv18DBw70v87ixYsVGxurhIQE/fznP5fb7dbUqVMlSfPnz9dtt92mRx55RA8++KCio6N18OBBbdq0SStXrlRSUpJmzJihBx54QM8995xuueUWHT16VKdOndK0adM0bNgwORwOvf3227rnnnsUGRn5lblnzJihf/qnf9Kzzz6rxx57TA8++KBKS0v129/+1p6/aACXZ/ciGQDma21ttVasWGHddNNNVlhYmBUXF2dNmTLF2rJli5WQkGAtXbrUv29LS4s1fvx4a9q0aZZl/d9C17feessaPXq0FR4ebmVkZFj79u0L+B07d+607rzzTqt///5WdHS0dfPNN1tLlizxb//rX/9qzZs3z7r++uut8PBwKyUlxVqzZo1/++LFi63rrrvOcjgc1owZM9rNvXXrVv/z3nrrLSslJcVyOp3WN7/5TWvNmjUspgUM4rCs/3/zAwDoBlu2bNHkyZN15swZfe1rX7M7DoBehhu+AQAAY1FUAACAsTj1AwAAjMURFQAAYCyKCgAAMBZFBQAAGIuiAgAAjEVRAQAAxqKoAAAAY1FUAACAsSgqAADAWP8PlnHdFd1q/joAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 59
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "17828f73c1d0b8a9"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}