bagging_and_boosting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "8c3755ff-60ea-4e0f-bb62-bd76499e6470",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training samples: 614, Testing samples: 154\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import random\n",
    "\n",
    "def load_dataset(file_path):\n",
    "    dataset = []\n",
    "    with open(file_path, 'r') as file:\n",
    "        csv_reader = csv.reader(file)\n",
    "        next(csv_reader)  \n",
    "        for row in csv_reader:\n",
    "            if not row:\n",
    "                continue\n",
    "            processed_row = [float(x) for x in row[:-1]] + [int(row[-1])]\n",
    "            dataset.append(processed_row)\n",
    "    return dataset\n",
    "\n",
    "def handle_missing_values(dataset, features):\n",
    "    for row in dataset:\n",
    "        for i in features:\n",
    "            if row[i] == 0:\n",
    "                feature_values = [r[i] for r in dataset if r[i] != 0]\n",
    "                mean_value = sum(feature_values) / len(feature_values)\n",
    "                row[i] = mean_value\n",
    "    return dataset\n",
    "\n",
    "def train_test_split(dataset, test_size=0.2):\n",
    "    random.shuffle(dataset)\n",
    "    split_index = int(len(dataset) * (1 - test_size))\n",
    "    return dataset[:split_index], dataset[split_index:]\n",
    "\n",
    "features_with_missing = [2, 3, 4, 5, 6]  \n",
    "\n",
    "file_path = r'C:\\Users\\adith\\Downloads\\pima-indians-diabetes.csv'\n",
    "dataset = load_dataset(file_path)\n",
    "dataset = handle_missing_values(dataset, features_with_missing)\n",
    "train_set, test_set = train_test_split(dataset, test_size=0.2)\n",
    "\n",
    "print(f\"Training samples: {len(train_set)}, Testing samples: {len(test_set)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "3e08c749-e161-448e-a565-da1ce86201a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "958263cf-0f79-4ebd-b475-e1c37f913606",
   "metadata": {},
   "outputs": [],
   "source": [
    "def entropy(dataset):\n",
    "    labels = [row[-1] for row in dataset]\n",
    "    total = len(labels)\n",
    "    counts = Counter(labels)\n",
    "    ent = 0.0\n",
    "    for count in counts.values():\n",
    "        probability = count / total\n",
    "        ent -= probability * math.log2(probability)\n",
    "    return ent\n",
    "\n",
    "def information_gain(parent, left, right):\n",
    "    total = len(parent)\n",
    "    ent_parent = entropy(parent)\n",
    "    ent_left = entropy(left)\n",
    "    ent_right = entropy(right)\n",
    "    weighted_ent = (len(left) / total) * ent_left + (len(right) / total) * ent_right\n",
    "    return ent_parent - weighted_ent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "ed5a4e60-89a8-4546-a57f-859dbe6e8c95",
   "metadata": {},
   "outputs": [],
   "source": [
    "def best_split(dataset, features):\n",
    "    best_feature = None\n",
    "    best_value = None\n",
    "    best_gain = -1\n",
    "    for feature in features:\n",
    "        values = set([row[feature] for row in dataset])\n",
    "        for value in values:\n",
    "            left = [row for row in dataset if row[feature] <= value]\n",
    "            right = [row for row in dataset if row[feature] > value]\n",
    "            if not left or not right:\n",
    "                continue\n",
    "            gain = information_gain(dataset, left, right)\n",
    "            if gain > best_gain:\n",
    "                best_gain = gain\n",
    "                best_feature = feature\n",
    "                best_value = value\n",
    "    return best_feature, best_value\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "217a5421-8a5f-4814-ba1d-48db215077f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_tree(dataset, features, depth=0, max_depth=10):\n",
    "    labels = [row[-1] for row in dataset]\n",
    "    if len(set(labels)) == 1:\n",
    "        return labels[0]\n",
    "    if not features or depth == max_depth:\n",
    "        return Counter(labels).most_common(1)[0][0]\n",
    "    best_feature, best_value = best_split(dataset, features)\n",
    "    if best_feature is None:\n",
    "        return Counter(labels).most_common(1)[0][0]\n",
    "    left = [row for row in dataset if row[best_feature] <= best_value]\n",
    "    right = [row for row in dataset if row[best_feature] > best_value]\n",
    "    if not left or not right:\n",
    "        return Counter(labels).most_common(1)[0][0]\n",
    "    tree = {\n",
    "        'feature': best_feature,\n",
    "        'value': best_value,\n",
    "        'left': build_tree(left, features, depth + 1, max_depth),\n",
    "        'right': build_tree(right, features, depth + 1, max_depth)\n",
    "    }\n",
    "    return tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "2f40fd41-b558-4380-879a-7a50789ba971",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(tree, instance):\n",
    "    if isinstance(tree, int):\n",
    "        return tree\n",
    "    feature = tree['feature']\n",
    "    value = tree['value']\n",
    "    if instance[feature] <= value:\n",
    "        return predict(tree['left'], instance)\n",
    "    else:\n",
    "        return predict(tree['right'], instance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "d31a6348-d423-4e39-b955-df43ae261d42",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bootstrap_sample(dataset):\n",
    "    n = len(dataset)\n",
    "    sample = []\n",
    "    for _ in range(n):\n",
    "        index = random.randint(0, n - 1)\n",
    "        sample.append(dataset[index])\n",
    "    return sample\n",
    "\n",
    "class BaggingClassifier:\n",
    "    def __init__(self, n_estimators=10, max_depth=10):\n",
    "        self.n_estimators = n_estimators\n",
    "        self.max_depth = max_depth\n",
    "        self.trees = []\n",
    "    \n",
    "    def fit(self, dataset, features):\n",
    "        for _ in range(self.n_estimators):\n",
    "            sample = bootstrap_sample(dataset)\n",
    "            tree = build_tree(sample, features, max_depth=self.max_depth)\n",
    "            self.trees.append(tree)\n",
    "    \n",
    "    def predict(self, instance):\n",
    "        predictions = [predict(tree, instance) for tree in self.trees]\n",
    "        return Counter(predictions).most_common(1)[0][0]\n",
    "    \n",
    "    def predict_batch(self, dataset):\n",
    "        return [self.predict(instance) for instance in dataset]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "bc512566-7afd-4a5f-b4cd-55863a9480c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdaBoostClassifier:\n",
    "    def __init__(self, n_estimators=10, max_depth=1):\n",
    "        self.n_estimators = n_estimators\n",
    "        self.max_depth = max_depth\n",
    "        self.trees = []\n",
    "        self.alpha = []\n",
    "    \n",
    "    def fit(self, dataset, features):\n",
    "        n = len(dataset)\n",
    "        weights = [1 / n] * n\n",
    "        for _ in range(self.n_estimators):\n",
    "            weighted_dataset = []\n",
    "            for i in range(n):\n",
    "                weighted_dataset.extend([dataset[i]] * int(weights[i] * 1000))  # Scale weights\n",
    "            tree = build_tree(weighted_dataset, features, max_depth=self.max_depth)\n",
    "            predictions = [predict(tree, row) for row in dataset]\n",
    "            error = 0\n",
    "            for i in range(n):\n",
    "                if predictions[i] != dataset[i][-1]:\n",
    "                    error += weights[i]\n",
    "            if error > 0.5:\n",
    "                break\n",
    "            alpha = 0.5 * math.log((1 - error) / (error + 1e-10))  # Add epsilon to avoid division by zero\n",
    "            self.alpha.append(alpha)\n",
    "            self.trees.append(tree)\n",
    "            # Update weights\n",
    "            for i in range(n):\n",
    "                if predictions[i] == dataset[i][-1]:\n",
    "                    weights[i] *= math.exp(-alpha)\n",
    "                else:\n",
    "                    weights[i] *= math.exp(alpha)\n",
    "            total_weight = sum(weights)\n",
    "            weights = [w / total_weight for w in weights]\n",
    "    \n",
    "    def predict(self, instance):\n",
    "        total = 0\n",
    "        for tree, alpha in zip(self.trees, self.alpha):\n",
    "            prediction = predict(tree, instance)\n",
    "            total += alpha if prediction == 1 else -alpha\n",
    "        return 1 if total >= 0 else 0\n",
    "    \n",
    "    def predict_batch(self, dataset):\n",
    "        return [self.predict(instance) for instance in dataset]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "3e9e9757-1361-4dbd-8c28-efaae8c1b0fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def majority_vote(predictions_list):\n",
    "    final_predictions = []\n",
    "    for i in range(len(predictions_list[0])):\n",
    "        votes = [predictions[i] for predictions in predictions_list]\n",
    "        final = Counter(votes).most_common(1)[0][0]\n",
    "        final_predictions.append(final)\n",
    "    return final_predictions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "4d3904bd-2718-4a2b-a9f6-aef757d44908",
   "metadata": {},
   "outputs": [],
   "source": [
    "def confusion_matrix(true_labels, predicted_labels):\n",
    "    TP = FP = TN = FN = 0\n",
    "    for true, pred in zip(true_labels, predicted_labels):\n",
    "        if true == 1 and pred == 1:\n",
    "            TP += 1\n",
    "        elif true == 0 and pred == 1:\n",
    "            FP += 1\n",
    "        elif true == 0 and pred == 0:\n",
    "            TN += 1\n",
    "        elif true == 1 and pred == 0:\n",
    "            FN += 1\n",
    "    return TP, FP, TN, FN\n",
    "\n",
    "def accuracy(true_labels, predicted_labels):\n",
    "    correct = sum(1 for true, pred in zip(true_labels, predicted_labels) if true == pred)\n",
    "    return correct / len(true_labels)\n",
    "\n",
    "def precision(true_labels, predicted_labels):\n",
    "    TP, FP, _, _ = confusion_matrix(true_labels, predicted_labels)\n",
    "    return TP / (TP + FP) if (TP + FP) != 0 else 0\n",
    "\n",
    "def recall(true_labels, predicted_labels):\n",
    "    TP, _, _, FN = confusion_matrix(true_labels, predicted_labels)\n",
    "    return TP / (TP + FN) if (TP + FN) != 0 else 0\n",
    "\n",
    "def f1_score(true_labels, predicted_labels):\n",
    "    prec = precision(true_labels, predicted_labels)\n",
    "    rec = recall(true_labels, predicted_labels)\n",
    "    return 2 * (prec * rec) / (prec + rec) if (prec + rec) != 0 else 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "30b58d8b-77eb-4c01-b1b3-7b67fae3e4a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_performance(true_labels, predicted_labels, model_name):\n",
    "    TP, FP, TN, FN = confusion_matrix(true_labels, predicted_labels)\n",
    "    acc = accuracy(true_labels, predicted_labels)\n",
    "    prec = precision(true_labels, predicted_labels)\n",
    "    rec = recall(true_labels, predicted_labels)\n",
    "    f1 = f1_score(true_labels, predicted_labels)\n",
    "    print(f\"Performance of {model_name}:\")\n",
    "    print(f\"Confusion Matrix: TP={TP}, FP={FP}, TN={TN}, FN={FN}\")\n",
    "    print(f\"Accuracy: {acc:.4f}\")\n",
    "    print(f\"Precision: {prec:.4f}\")\n",
    "    print(f\"Recall: {rec:.4f}\")\n",
    "    print(f\"F1 Score: {f1:.4f}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "b7c42338-5b34-4c08-9d75-81dde9d4684e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of Conventional Decision Tree:\n",
      "Confusion Matrix: TP=28, FP=18, TN=80, FN=28\n",
      "Accuracy: 0.7013\n",
      "Precision: 0.6087\n",
      "Recall: 0.5000\n",
      "F1 Score: 0.5490\n",
      "\n"
     ]
    }
   ],
   "source": [
    "features = list(range(len(dataset[0]) - 1))\n",
    "\n",
    "conventional_tree = build_tree(train_set, features, max_depth=10)\n",
    "conventional_predictions = [predict(conventional_tree, instance) for instance in test_set]\n",
    "true_labels = [row[-1] for row in test_set]\n",
    "print_performance(true_labels, conventional_predictions, \"Conventional Decision Tree\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "b84086c7-e165-4fb6-a95f-2a5cbeb3993d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of Bagging Ensemble:\n",
      "Confusion Matrix: TP=35, FP=15, TN=83, FN=21\n",
      "Accuracy: 0.7662\n",
      "Precision: 0.7000\n",
      "Recall: 0.6250\n",
      "F1 Score: 0.6604\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bagging = BaggingClassifier(n_estimators=10, max_depth=10)\n",
    "bagging.fit(train_set, features)\n",
    "bagging_predictions = bagging.predict_batch(test_set)\n",
    "print_performance(true_labels, bagging_predictions, \"Bagging Ensemble\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "94eb1fd6-6e6e-4efc-a7bc-4b034c720e2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of Boosting Ensemble:\n",
      "Confusion Matrix: TP=29, FP=20, TN=78, FN=27\n",
      "Accuracy: 0.6948\n",
      "Precision: 0.5918\n",
      "Recall: 0.5179\n",
      "F1 Score: 0.5524\n",
      "\n"
     ]
    }
   ],
   "source": [
    "boosting = AdaBoostClassifier(n_estimators=10, max_depth=1)\n",
    "boosting.fit(train_set, features)\n",
    "boosting_predictions = boosting.predict_batch(test_set)\n",
    "print_performance(true_labels, boosting_predictions, \"Boosting Ensemble\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "1c96ded5-d232-4cbf-8fdb-7c39eb54de04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of Majority Voting Ensemble:\n",
      "Confusion Matrix: TP=35, FP=15, TN=83, FN=21\n",
      "Accuracy: 0.7662\n",
      "Precision: 0.7000\n",
      "Recall: 0.6250\n",
      "F1 Score: 0.6604\n",
      "\n"
     ]
    }
   ],
   "source": [
    "votes = [bagging_predictions, boosting_predictions]\n",
    "voted_predictions = majority_vote(votes)\n",
    "print_performance(true_labels, voted_predictions, \"Majority Voting Ensemble\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "518e2251-1b24-4c81-961d-867e0e3189ab",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}