Naive_GPT_4o_mini.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOZPvb7svz0nsLtbMynWlfu"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Naive GPT-4o-mini"
      ],
      "metadata": {
        "id": "h3PIxpMp8toe"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Preparation"
      ],
      "metadata": {
        "id": "pZ8j-uZX5F6k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade openai"
      ],
      "metadata": {
        "id": "NT0ffTZC_IKo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "df5d88ce-ec7e-4c29-9548-4fa9b25308e3",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (1.54.4)\n",
            "Collecting openai\n",
            "  Downloading openai-1.55.3-py3-none-any.whl.metadata (24 kB)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from openai) (1.9.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai) (0.27.2)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai) (0.7.1)\n",
            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.9.2)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.6)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.11 in /usr/local/lib/python3.10/dist-packages (from openai) (4.12.2)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.10)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2024.8.30)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (1.0.7)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.23.4)\n",
            "Downloading openai-1.55.3-py3-none-any.whl (389 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m389.6/389.6 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: openai\n",
            "  Attempting uninstall: openai\n",
            "    Found existing installation: openai 1.54.4\n",
            "    Uninstalling openai-1.54.4:\n",
            "      Successfully uninstalled openai-1.54.4\n",
            "Successfully installed openai-1.55.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "import json\n",
        "import os\n",
        "from openai import OpenAI\n",
        "from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, f1_score, recall_score"
      ],
      "metadata": {
        "id": "dRKqmvRJ-8P2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if not os.path.exists(\"data\"):\n",
        "  !pip install github-clone\n",
        "  !ghclone https://github.com/yiw008/nondet-project/tree/main/data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Mg49aaAyIr8",
        "outputId": "33407264-3747-42a4-b2aa-7a06f3499f0b",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting github-clone\n",
            "  Downloading github_clone-1.2.0-py3-none-any.whl.metadata (3.7 kB)\n",
            "Requirement already satisfied: requests>=2.20.0 in /usr/local/lib/python3.10/dist-packages (from github-clone) (2.32.3)\n",
            "Collecting docopt>=0.6.2 (from github-clone)\n",
            "  Downloading docopt-0.6.2.tar.gz (25 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20.0->github-clone) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20.0->github-clone) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20.0->github-clone) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20.0->github-clone) (2024.8.30)\n",
            "Downloading github_clone-1.2.0-py3-none-any.whl (9.1 kB)\n",
            "Building wheels for collected packages: docopt\n",
            "  Building wheel for docopt (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13706 sha256=d06029fe79b7029befea440e4bd445d1ce620ad99992c1f52541b54264cdd673\n",
            "  Stored in directory: /root/.cache/pip/wheels/fc/ab/d4/5da2067ac95b36618c629a5f93f809425700506f72c9732fac\n",
            "Successfully built docopt\n",
            "Installing collected packages: docopt, github-clone\n",
            "Successfully installed docopt-0.6.2 github-clone-1.2.0\n",
            "Cloning into 'data'...\n",
            "done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def str_to_bool(string):\n",
        "  if string == \"True\":\n",
        "    return True\n",
        "  elif string == \"False\":\n",
        "    return False\n",
        "  return False"
      ],
      "metadata": {
        "id": "VWdLFQKGChBC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "api_key = \"\" # TODO\n",
        "os.environ['OPENAI_API_KEY'] = api_key"
      ],
      "metadata": {
        "id": "5TYkF0KAxBOT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "client = OpenAI()\n",
        "our_model = \"gpt-4o-mini-2024-07-18\""
      ],
      "metadata": {
        "id": "5dpAvb03xB9h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def test_project(project_name):\n",
        "  print(f\"Test project: {project_name}\")\n",
        "  test_set = []\n",
        "  y_test = []\n",
        "  with open(f\"data/{project_name}/test_set.jsonl\", \"r\") as file:\n",
        "    for line in file:\n",
        "      data = json.loads(line)\n",
        "      test_set.append(data['messages'])\n",
        "      y_test.append(str_to_bool(data['messages'][2]['content']))\n",
        "\n",
        "  y_pred = []\n",
        "  start = time.time()\n",
        "\n",
        "  for i in range(len(test_set)):\n",
        "    completion = client.chat.completions.create(\n",
        "      model=our_model,\n",
        "      messages=test_set[i]\n",
        "    )\n",
        "    y_pred.append(str_to_bool(completion.choices[0].message.content))\n",
        "\n",
        "  end = time.time()\n",
        "  print(f\"Testing time: {end - start:.4f} seconds\")\n",
        "\n",
        "  print(\"Y_test:\")\n",
        "  print(y_test)\n",
        "  print(\"Y_pred:\")\n",
        "  print(y_pred)\n",
        "\n",
        "  confusion_matrix_res = confusion_matrix(y_test, y_pred, labels=[False, True])\n",
        "  print(\"Confusion Matrix:\")\n",
        "  print(confusion_matrix_res)\n",
        "\n",
        "  accuracy = accuracy_score(y_test, y_pred)\n",
        "  print(f\"Accuracy: {accuracy}\")\n",
        "\n",
        "  precision = precision_score(y_test, y_pred)\n",
        "  print(f\"Precision: {precision}\")\n",
        "\n",
        "  f1 = f1_score(y_test, y_pred)\n",
        "  print(f\"F1 Score: {f1}\")\n",
        "\n",
        "  recall = recall_score(y_test, y_pred)\n",
        "  print(f\"Recall: {recall}\")\n",
        "\n",
        "  return accuracy, precision, f1, recall"
      ],
      "metadata": {
        "id": "ViKCZJ6-xDmh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Let's Go"
      ],
      "metadata": {
        "id": "7xUFkxIG33Ev"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values = [0] * 10\n",
        "precision_values = [0] * 10\n",
        "f1_values = [0] * 10\n",
        "recall_values = [0] * 10"
      ],
      "metadata": {
        "id": "zVhCE1JCaot8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[0], precision_values[0], f1_values[0], recall_values[0] = test_project(\"Butter.MAS.PythonAPI\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c2k2gEam2hHt",
        "outputId": "da10d73c-c081-4cb3-b2f9-72343ed753d1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: Butter.MAS.PythonAPI\n",
            "Testing time: 26.0813 seconds\n",
            "Y_test:\n",
            "[True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, True, True, False, True, True, True, False, True, False, False, False, True, True, False, True, True, True, False, False, True, False]\n",
            "Y_pred:\n",
            "[True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, True, True, False, True, True, True, False, True, False, False, False, True, True, False, True, True, True, False, False, True, False]\n",
            "Confusion Matrix:\n",
            "[[11  0]\n",
            " [ 0 53]]\n",
            "Accuracy: 1.0\n",
            "Precision: 1.0\n",
            "F1 Score: 1.0\n",
            "Recall: 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[1], precision_values[1], f1_values[1], recall_values[1] = test_project(\"flask-multi-redis\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QPIThfQ9aZG4",
        "outputId": "2054306a-52ea-4e30-ac28-97a2db184af1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: flask-multi-redis\n",
            "Testing time: 13.7050 seconds\n",
            "Y_test:\n",
            "[True, True, True, True, True, True, True, True, True, False, True, True, False, True, True, True, False, False, False, True, True, True, True, False, True, True, True, True, False, True, True, True, True]\n",
            "Y_pred:\n",
            "[False, True, True, True, True, True, False, True, True, False, True, True, False, True, True, True, False, False, False, False, True, True, True, False, True, True, True, True, False, True, True, True, True]\n",
            "Confusion Matrix:\n",
            "[[ 7  0]\n",
            " [ 3 23]]\n",
            "Accuracy: 0.9090909090909091\n",
            "Precision: 1.0\n",
            "F1 Score: 0.9387755102040817\n",
            "Recall: 0.8846153846153846\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[2], precision_values[2], f1_values[2], recall_values[2] = test_project(\"centreon-sdk-python\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "onZormhr2lVw",
        "outputId": "f28762cd-f809-44cd-efcb-5f97a1e7fb86"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: centreon-sdk-python\n",
            "Testing time: 18.2732 seconds\n",
            "Y_test:\n",
            "[True, True, True, True, True, True, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True]\n",
            "Y_pred:\n",
            "[True, True, True, True, True, True, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True]\n",
            "Confusion Matrix:\n",
            "[[ 1  0]\n",
            " [ 0 41]]\n",
            "Accuracy: 1.0\n",
            "Precision: 1.0\n",
            "F1 Score: 1.0\n",
            "Recall: 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[3], precision_values[3], f1_values[3], recall_values[3] = test_project(\"cloudnetpy\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nJxoyNrf2yHC",
        "outputId": "d1bfbec1-0ac6-4d79-f349-52a3c7c1ef8d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: cloudnetpy\n",
            "Testing time: 195.5107 seconds\n",
            "Y_test:\n",
            "[False, True, True, True, True, True, False, True, True, False, False, False, False, False, True, False, False, False, True, True, False, False, True, False, False, False, False, False, True, False, False, False, False, False, True, False, False, False, False, True, True, False, False, False, False, True, True, False, True, False, False, False, False, False, True, False, False, False, False, True, True, False, False, True, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, True, False, False, False, True, False, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, False, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, True, False, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False]\n",
            "Y_pred:\n",
            "[False, True, True, True, True, True, False, True, True, False, False, False, False, False, True, False, False, False, True, True, False, False, True, False, False, False, False, False, True, False, False, False, False, False, True, False, False, False, False, True, True, False, False, False, False, True, True, False, True, False, False, False, False, False, True, False, False, False, False, True, True, False, False, True, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, True, False, False, False, True, False, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, False, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, True, False, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False]\n",
            "Confusion Matrix:\n",
            "[[246   0]\n",
            " [  0  51]]\n",
            "Accuracy: 1.0\n",
            "Precision: 1.0\n",
            "F1 Score: 1.0\n",
            "Recall: 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[4], precision_values[4], f1_values[4], recall_values[4] = test_project(\"crom\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_2s4iQym20jO",
        "outputId": "e465b7a8-e79c-4e8f-b714-8404a6162375"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: crom\n",
            "Testing time: 35.8987 seconds\n",
            "Y_test:\n",
            "[True, False, False, False, True, False, True, False, True, False, True, False, False, False, True, False, False, True, True, False, True, False, False, False, True, True, False, False, True, True, False, False, False, True, True, False, True, True, True, False, True, False, True, False, True, True, False, False, True, True, False, False, False, True, True, False, False, True, True, False, False, False, True, False, False, True, False, False, False, False, False, False, False, True, True, True, False, True, True, True, False, True, False, True]\n",
            "Y_pred:\n",
            "[True, False, False, False, False, False, True, False, True, False, True, False, False, False, True, False, False, True, True, False, True, False, False, False, True, True, False, False, True, True, False, False, False, True, False, False, True, False, True, False, True, False, True, False, True, True, False, False, True, True, False, False, False, True, True, False, False, True, True, False, False, False, True, False, False, True, False, False, False, False, False, False, False, True, True, False, False, True, True, True, False, True, False, True]\n",
            "Confusion Matrix:\n",
            "[[46  0]\n",
            " [ 4 34]]\n",
            "Accuracy: 0.9523809523809523\n",
            "Precision: 1.0\n",
            "F1 Score: 0.9444444444444444\n",
            "Recall: 0.8947368421052632\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[5], precision_values[5], f1_values[5], recall_values[5] = test_project(\"easypy\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x4qlFACs22V8",
        "outputId": "7e1743e2-c9f7-4c00-8520-779e576437c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: easypy\n",
            "Testing time: 134.3147 seconds\n",
            "Y_test:\n",
            "[False, False, False, False, False, False, False, False, False, False, True, False, True, False, False, False, False, False, True, True, True, False, False, True, False, True, False, False, True, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, True, False, True, False, False, False, True, True, False, True, True, False, False, True, True, False, False, False, True, True, False, False, False, True, False, False, False, True, False, False, False, True, True, False, False, False, True, False, False, False, False, False, False, True, False, False, False, True, False, False, True, False, True, False, False, False, False, True, False, False, False, False, False, False, True, False, True, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, True, True, False, True, False, False, False, True, True, False, True, True, False, True, False, False, False, False, False, False, False, False, False]\n",
            "Y_pred:\n",
            "[False, False, False, False, False, False, False, False, False, False, True, False, True, False, False, False, False, False, True, True, True, False, False, False, False, True, False, False, True, True, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, True, False, False, False, True, False, False, True, True, False, False, False, True, False, False, False, True, True, False, False, False, True, False, False, False, True, False, False, False, True, True, False, False, False, True, True, False, False, False, False, False, True, False, False, False, True, False, False, True, False, True, False, False, False, False, True, False, False, False, False, False, False, True, False, True, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, True, True, True, True, False, False, False, True, True, False, True, True, False, True, False, False, False, False, False, False, True, False, False]\n",
            "Confusion Matrix:\n",
            "[[126   4]\n",
            " [  5  39]]\n",
            "Accuracy: 0.9482758620689655\n",
            "Precision: 0.9069767441860465\n",
            "F1 Score: 0.896551724137931\n",
            "Recall: 0.8863636363636364\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[6], precision_values[6], f1_values[6], recall_values[6] = test_project(\"eppy\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kBB_nmRz24BR",
        "outputId": "36c970df-4136-4f83-e6fa-68464a02c60f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: eppy\n",
            "Testing time: 93.7987 seconds\n",
            "Y_test:\n",
            "[True, True, False, True, False, True, False, True, False, False, False, True, False, False, True, False, False, True, False, True, True, False, False, False, False, False, True, False, False, False, True, False, False, True, True, True, True, True, False, False, False, True, True, True, False, False, False, True, False, False, True, True, False, False, True, True, False, True, True, False, True, True, False, False, False, False, False, False, True, True, True, False, True, False, True, False, False, False, True, True, False, False, True, False, True, False, False, True, False, True, False, True, False, False, False, True, True, False, True, False, True, True, True, False, False, False, False, False, True, False, True, True, False, True, True, False, False, False, True, True, True, False, True, False, False, True, True, False, False, False, False, False, False, False, False, True, False, False, False, False, True, True, True, True, False, True, True, False, True, False, True, False, False, False, True, True, False, True, False, False, False, False, False, False, False, True, True, True, False, False, False, False, True, True, True, True, False, True, True, False, False, False, False, True, True, False, False, False, False, False, False, True, True, True]\n",
            "Y_pred:\n",
            "[True, True, False, True, False, True, False, True, False, False, False, True, False, False, False, False, False, True, False, True, True, False, False, False, False, False, True, False, False, False, True, False, False, True, True, True, True, True, False, False, False, False, True, True, False, False, False, True, False, False, True, True, False, False, True, True, False, True, True, False, False, True, False, False, False, False, False, False, True, True, True, False, True, False, True, False, True, False, True, True, False, False, True, False, True, True, False, True, False, True, False, True, False, False, False, True, True, False, False, False, True, True, True, False, False, False, False, False, True, False, True, True, False, True, True, False, False, False, True, True, True, False, True, False, False, True, False, False, False, False, False, False, False, True, False, True, False, False, False, False, False, True, True, True, False, True, True, False, True, False, True, False, False, False, False, True, False, True, False, False, False, False, False, False, False, True, True, True, False, False, False, False, True, True, True, True, False, True, True, False, False, False, False, True, True, False, False, False, False, False, False, True, True, True]\n",
            "Confusion Matrix:\n",
            "[[107   3]\n",
            " [  7  77]]\n",
            "Accuracy: 0.9484536082474226\n",
            "Precision: 0.9625\n",
            "F1 Score: 0.9390243902439024\n",
            "Recall: 0.9166666666666666\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[7], precision_values[7], f1_values[7], recall_values[7] = test_project(\"pykicad\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "azQbX0eJaeTz",
        "outputId": "4e0481a4-ab1a-409c-ab4d-f874621f9658"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: pykicad\n",
            "Testing time: 21.3315 seconds\n",
            "Y_test:\n",
            "[False, False, False, True, False, True, True, True, False, True, True, False, True, True, False, False, False, False, True, True, True, True, True, True, False, True, False, True, False, False, True, True, True, False, False, False, True, True, True, False, True, True, True, True, True, True, False, False]\n",
            "Y_pred:\n",
            "[False, False, False, True, False, False, True, True, False, False, False, False, False, False, False, False, False, False, True, False, True, True, True, True, False, True, False, True, False, False, True, True, False, False, False, False, True, False, True, False, False, True, True, True, True, True, False, False]\n",
            "Confusion Matrix:\n",
            "[[20  0]\n",
            " [ 9 19]]\n",
            "Accuracy: 0.8125\n",
            "Precision: 1.0\n",
            "F1 Score: 0.8085106382978723\n",
            "Recall: 0.6785714285714286\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[8], precision_values[8], f1_values[8], recall_values[8] = test_project(\"reframe\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yGR4iPBS25sS",
        "outputId": "a3dcc9e5-e904-4600-bb2c-41a7df624890"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: reframe\n",
            "Testing time: 318.7577 seconds\n",
            "Y_test:\n",
            "[False, True, False, False, False, False, True, False, True, False, False, False, False, False, False, False, True, True, False, True, False, False, False, False, False, False, False, False, True, False, False, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, True, False, True, False, False, False, False, False, False, False, True, False, True, True, False, False, False, False, False, True, True, True, True, True, False, False, False, False, False, False, False, False, False, True, False, True, False, True, False, True, False, True, True, True, False, False, False, False, False, False, False, False, False, False, False, True, True, False, True, False, False, False, False, False, True, False, True, False, False, False, True, False, True, True, False, False, False, False, True, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, True, True, False, False, True, False, False, False, True, False, False, False, False, False, False, False, True, False, False, False, False, False, False, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, True, False, False, False, False, False, False, False, True, True, False, True, True, False, True, False, False, False, True, True, False, False, True, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, False, True, False, True, False, False, True, False, False, False, False, True, True, False, False, False, True, True, False, False, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, True, True, False, False, False, True, True, True, False, True, True, False, True, False, False, False, False, False, False, False, True, False, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, False, True, False, False, False, False, False, True, True, False, True, False, True, False, True, True, False, False, False, True, False, False, False, True, False, False, True, False, False, True, False, False, True, False, False, True, True, False, False, False, False, False, False, False, False, True, False, False, False, False, True, False, False, False, False, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, False, True, True, False, True, False, False, False, False, False, False, False, False, False, True, False, False, False, False, True, False, False, False, False, True, False, False, True, False, False, False, False, False, True, False, False, False, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, False, False, False, False, False, False, True, False, False, False, False, False, False, False, True, False, True, False, False, False, False, True, False, False, True, False, False, True, False, False, False, False, True, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, True, False, True, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, False, False, False, False, False]\n",
            "Y_pred:\n",
            "[False, True, False, False, False, False, True, False, True, True, False, False, False, False, False, False, True, True, False, True, False, False, False, False, False, False, False, False, True, False, False, True, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, True, False, True, False, False, False, False, False, False, False, True, False, True, True, False, False, False, False, False, True, True, True, True, True, False, False, False, False, False, False, False, False, False, True, False, True, False, True, False, True, False, True, True, True, False, False, False, False, False, False, False, False, False, False, False, True, True, False, True, False, False, False, False, True, True, False, True, False, False, False, True, False, True, True, False, False, False, False, True, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, True, True, False, False, True, False, False, False, True, False, False, False, False, False, False, False, True, False, False, False, False, False, True, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, True, False, False, False, False, False, False, False, True, True, False, True, True, False, True, False, False, False, True, True, False, False, True, False, False, False, True, True, False, False, True, False, False, False, False, False, False, False, False, True, False, True, False, False, True, False, False, False, False, True, True, False, False, False, True, True, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, True, True, False, False, False, True, True, True, False, True, True, False, True, False, False, False, False, False, False, False, True, False, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, False, True, False, False, False, False, False, True, True, False, True, False, True, False, True, True, False, False, False, True, False, False, False, True, False, False, True, False, False, True, False, False, True, False, False, True, True, False, False, False, False, False, False, False, False, True, False, False, False, False, True, False, False, False, False, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, True, True, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, False, True, True, False, True, False, False, False, False, False, False, False, False, False, True, False, False, False, False, True, False, False, False, False, True, False, False, True, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, False, False, False, False, False, False, True, True, False, False, False, False, False, False, True, False, True, False, False, False, False, True, False, False, True, False, False, True, False, False, False, False, True, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, True, False, True, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, False, False, False, False, False]\n",
            "Confusion Matrix:\n",
            "[[458   7]\n",
            " [  2 134]]\n",
            "Accuracy: 0.9850249584026622\n",
            "Precision: 0.950354609929078\n",
            "F1 Score: 0.9675090252707581\n",
            "Recall: 0.9852941176470589\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_values[9], precision_values[9], f1_values[9], recall_values[9] = test_project(\"webssh\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1HNylCkBaiHP",
        "outputId": "21523a40-5291-4c25-b9e3-8ea2f024bc87"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test project: webssh\n",
            "Testing time: 34.8538 seconds\n",
            "Y_test:\n",
            "[True, False, False, True, False, False, False, False, True, True, False, False, True, True, False, True, False, False, True, False, False, False, False, False, True, False, True, True, False, True, False, False, False, True, False, False, False, False, False, True, False, True, False, False, False, False, False, False, False, True, False, False, True, False, True, True, False, False, False, False, False, False, True, False, False, True, False, False, False, True, True, False, True, False, False, False, False, False, False, True, False, False, False, True, True, False, True, False, True]\n",
            "Y_pred:\n",
            "[True, False, True, True, False, False, False, False, True, True, False, False, True, True, False, True, False, False, True, False, False, True, False, False, True, False, True, True, False, True, False, False, False, True, False, False, False, False, False, True, False, True, False, False, False, False, False, True, False, True, False, False, False, False, True, True, False, False, False, False, False, False, True, False, False, True, False, False, False, True, True, False, True, False, False, False, False, False, False, True, False, False, False, True, True, False, True, False, True]\n",
            "Confusion Matrix:\n",
            "[[57  3]\n",
            " [ 1 28]]\n",
            "Accuracy: 0.9550561797752809\n",
            "Precision: 0.9032258064516129\n",
            "F1 Score: 0.9333333333333333\n",
            "Recall: 0.9655172413793104\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "avg_accuracy = sum(accuracy_values) / len(accuracy_values)\n",
        "avg_precision = sum(precision_values) / len(precision_values)\n",
        "avg_f1 = sum(f1_values) / len(f1_values)\n",
        "avg_recall = sum(recall_values) / len(recall_values)\n",
        "\n",
        "print(f\"Average Accuracy: {avg_accuracy}\")\n",
        "print(f\"Average Precision: {avg_precision}\")\n",
        "print(f\"Average F1 Score: {avg_f1}\")\n",
        "print(f\"Average Recall: {avg_recall}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OdFslKdJchn3",
        "outputId": "927cb404-791d-4c37-c695-7684c0bccc3d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Accuracy: 0.9510782469966192\n",
            "Average Precision: 0.9723057160566737\n",
            "Average F1 Score: 0.9428149065932324\n",
            "Average Recall: 0.921176531734875\n"
          ]
        }
      ]
    }
  ]
}