count-lg-good.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fd76fba4",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:25.250063Z",
     "iopub.status.busy": "2022-11-28T03:56:25.249578Z",
     "iopub.status.idle": "2022-11-28T03:56:26.731696Z",
     "shell.execute_reply": "2022-11-28T03:56:26.730602Z"
    },
    "papermill": {
     "duration": 1.494378,
     "end_time": "2022-11-28T03:56:26.734932",
     "exception": false,
     "start_time": "2022-11-28T03:56:25.240554",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.stem import PorterStemmer #to find out the base root of the words \n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer #find TF*IDF  (Term Frequency(on one by one sentence ) = rep/total word , Inverse Document Frequency(on whole dataset) = log(total sen/rep))\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score#cration histogram ,count frequency ,make all words as feature\n",
    "# from nltk.metrics.distance import jaccard_distance\n",
    "# from nltk.util import ngrams\n",
    "# from nltk.metrics.distance  import edit_distance\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "10829bfa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:26.748334Z",
     "iopub.status.busy": "2022-11-28T03:56:26.747957Z",
     "iopub.status.idle": "2022-11-28T03:56:27.132933Z",
     "shell.execute_reply": "2022-11-28T03:56:27.131723Z"
    },
    "papermill": {
     "duration": 0.3963,
     "end_time": "2022-11-28T03:56:27.137280",
     "exception": false,
     "start_time": "2022-11-28T03:56:26.740980",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /usr/share/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /usr/share/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package words to /usr/share/nltk_data...\n",
      "[nltk_data]   Package words is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')  \n",
    "nltk.download('stopwords')\n",
    "nltk.download('words')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a7014459",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:27.150765Z",
     "iopub.status.busy": "2022-11-28T03:56:27.150342Z",
     "iopub.status.idle": "2022-11-28T03:56:31.895019Z",
     "shell.execute_reply": "2022-11-28T03:56:31.893731Z"
    },
    "papermill": {
     "duration": 4.754752,
     "end_time": "2022-11-28T03:56:31.897991",
     "exception": false,
     "start_time": "2022-11-28T03:56:27.143239",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"../input/yahoo-troll-question-detection/train_df.csv\")\n",
    "dt = pd.read_csv(\"../input/yahoo-troll-question-detection/test_df.csv\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d8f2c13b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:31.914386Z",
     "iopub.status.busy": "2022-11-28T03:56:31.913051Z",
     "iopub.status.idle": "2022-11-28T03:56:31.918803Z",
     "shell.execute_reply": "2022-11-28T03:56:31.917725Z"
    },
    "papermill": {
     "duration": 0.016214,
     "end_time": "2022-11-28T03:56:31.921218",
     "exception": false,
     "start_time": "2022-11-28T03:56:31.905004",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "whole_input_data_after_preprocessing=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "70953fcf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:31.934508Z",
     "iopub.status.busy": "2022-11-28T03:56:31.934122Z",
     "iopub.status.idle": "2022-11-28T03:56:31.943204Z",
     "shell.execute_reply": "2022-11-28T03:56:31.942134Z"
    },
    "papermill": {
     "duration": 0.018643,
     "end_time": "2022-11-28T03:56:31.945685",
     "exception": false,
     "start_time": "2022-11-28T03:56:31.927042",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def Preprocessing_of_data(start,end):\n",
    "#     ps=PorterStemmer()\n",
    "    for i in range(start,end):  \n",
    "        words=re.sub('[^a-zA-Z]',' ',df['question_text'][i]) #removing of all other char other then alphabet a to z and A to Z \n",
    "        words=words.lower() #convert all words to the lowercase\n",
    "#         words=words.split()\n",
    "#         words=getCorrectWords(words)\n",
    "#         words=[ps.stem(x) for x in words if x  not in set(stopwords.words('english')) ]\n",
    "#         words=' '.join(words)\n",
    "        whole_input_data_after_preprocessing.append(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6109da53",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:31.959250Z",
     "iopub.status.busy": "2022-11-28T03:56:31.958418Z",
     "iopub.status.idle": "2022-11-28T03:56:44.254955Z",
     "shell.execute_reply": "2022-11-28T03:56:44.254074Z"
    },
    "papermill": {
     "duration": 12.30596,
     "end_time": "2022-11-28T03:56:44.257425",
     "exception": false,
     "start_time": "2022-11-28T03:56:31.951465",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "Preprocessing_of_data(0,1000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bc627af9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:44.271235Z",
     "iopub.status.busy": "2022-11-28T03:56:44.270589Z",
     "iopub.status.idle": "2022-11-28T03:56:44.275919Z",
     "shell.execute_reply": "2022-11-28T03:56:44.275183Z"
    },
    "papermill": {
     "duration": 0.014512,
     "end_time": "2022-11-28T03:56:44.277856",
     "exception": false,
     "start_time": "2022-11-28T03:56:44.263344",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(whole_input_data_after_preprocessing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "86abceb4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:44.292692Z",
     "iopub.status.busy": "2022-11-28T03:56:44.292262Z",
     "iopub.status.idle": "2022-11-28T03:56:48.093373Z",
     "shell.execute_reply": "2022-11-28T03:56:48.092223Z"
    },
    "papermill": {
     "duration": 3.812634,
     "end_time": "2022-11-28T03:56:48.096308",
     "exception": false,
     "start_time": "2022-11-28T03:56:44.283674",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "main_test_input=[]\n",
    "# ps=PorterStemmer()\n",
    "for i in range(0,len(dt)):  \n",
    "    m_words=re.sub('[^a-zA-Z]',' ',dt['question_text'][i]) #removing of all other char other then alphabet a to z and A to Z \n",
    "    m_words=m_words.lower() #convert all words to the lowercase\n",
    "#     m_words=m_words.split()\n",
    "#     m_words=[ps.stem(x) for x in m_words if x  not in set(stopwords.words('english')) ]\n",
    "#     m_words=' '.join(m_words)\n",
    "    main_test_input.append(m_words)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3c7ac892",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:48.109752Z",
     "iopub.status.busy": "2022-11-28T03:56:48.109350Z",
     "iopub.status.idle": "2022-11-28T03:56:48.115190Z",
     "shell.execute_reply": "2022-11-28T03:56:48.114469Z"
    },
    "papermill": {
     "duration": 0.014774,
     "end_time": "2022-11-28T03:56:48.117115",
     "exception": false,
     "start_time": "2022-11-28T03:56:48.102341",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "306122"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(main_test_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8395b0fc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:48.130727Z",
     "iopub.status.busy": "2022-11-28T03:56:48.129988Z",
     "iopub.status.idle": "2022-11-28T03:56:48.152091Z",
     "shell.execute_reply": "2022-11-28T03:56:48.151285Z"
    },
    "papermill": {
     "duration": 0.031413,
     "end_time": "2022-11-28T03:56:48.154500",
     "exception": false,
     "start_time": "2022-11-28T03:56:48.123087",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tr_i=whole_input_data_after_preprocessing\n",
    "te_i=main_test_input\n",
    "combination_full_test=tr_i+te_i\n",
    "y=df['target']\n",
    "y=y[:len(tr_i)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9a007b04",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:56:48.168897Z",
     "iopub.status.busy": "2022-11-28T03:56:48.168087Z",
     "iopub.status.idle": "2022-11-28T03:57:06.435118Z",
     "shell.execute_reply": "2022-11-28T03:57:06.433943Z"
    },
    "papermill": {
     "duration": 18.276982,
     "end_time": "2022-11-28T03:57:06.437677",
     "exception": false,
     "start_time": "2022-11-28T03:56:48.160695",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "main_250000_cv=CountVectorizer()\n",
    "till_full_TFIDF_X=main_250000_cv.fit_transform(combination_full_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2b456206",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:57:06.451412Z",
     "iopub.status.busy": "2022-11-28T03:57:06.451013Z",
     "iopub.status.idle": "2022-11-28T03:57:06.456245Z",
     "shell.execute_reply": "2022-11-28T03:57:06.455145Z"
    },
    "papermill": {
     "duration": 0.014725,
     "end_time": "2022-11-28T03:57:06.458510",
     "exception": false,
     "start_time": "2022-11-28T03:57:06.443785",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1306122, 178847)\n",
      "1000000\n"
     ]
    }
   ],
   "source": [
    "print(till_full_TFIDF_X.shape)\n",
    "print(len(y))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8ccb5796",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:57:06.472233Z",
     "iopub.status.busy": "2022-11-28T03:57:06.471855Z",
     "iopub.status.idle": "2022-11-28T03:57:06.802375Z",
     "shell.execute_reply": "2022-11-28T03:57:06.801098Z"
    },
    "papermill": {
     "duration": 0.340562,
     "end_time": "2022-11-28T03:57:06.805055",
     "exception": false,
     "start_time": "2022-11-28T03:57:06.464493",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "main_full_X_train=till_full_TFIDF_X[:1000000]\n",
    "main_full_X_test=till_full_TFIDF_X[1000000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c6f98300",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:57:06.818544Z",
     "iopub.status.busy": "2022-11-28T03:57:06.818171Z",
     "iopub.status.idle": "2022-11-28T03:57:06.823255Z",
     "shell.execute_reply": "2022-11-28T03:57:06.822417Z"
    },
    "papermill": {
     "duration": 0.014288,
     "end_time": "2022-11-28T03:57:06.825343",
     "exception": false,
     "start_time": "2022-11-28T03:57:06.811055",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000000, 178847)\n",
      "(306122, 178847)\n"
     ]
    }
   ],
   "source": [
    "print(main_full_X_train.shape)\n",
    "print(main_full_X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5cbaacbf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:57:06.838398Z",
     "iopub.status.busy": "2022-11-28T03:57:06.838039Z",
     "iopub.status.idle": "2022-11-28T03:57:06.842856Z",
     "shell.execute_reply": "2022-11-28T03:57:06.841804Z"
    },
    "papermill": {
     "duration": 0.013804,
     "end_time": "2022-11-28T03:57:06.844938",
     "exception": false,
     "start_time": "2022-11-28T03:57:06.831134",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "# lr = LogisticRegression(max_iter=200)\n",
    "# main_full_yahoo_model=lr.fit(main_full_X_train,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "921a95ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:57:06.858339Z",
     "iopub.status.busy": "2022-11-28T03:57:06.857966Z",
     "iopub.status.idle": "2022-11-28T03:57:06.863428Z",
     "shell.execute_reply": "2022-11-28T03:57:06.862183Z"
    },
    "papermill": {
     "duration": 0.014579,
     "end_time": "2022-11-28T03:57:06.865573",
     "exception": false,
     "start_time": "2022-11-28T03:57:06.850994",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "logModel = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "060254f0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:57:06.878779Z",
     "iopub.status.busy": "2022-11-28T03:57:06.878390Z",
     "iopub.status.idle": "2022-11-28T03:57:06.883121Z",
     "shell.execute_reply": "2022-11-28T03:57:06.882034Z"
    },
    "papermill": {
     "duration": 0.013764,
     "end_time": "2022-11-28T03:57:06.885209",
     "exception": false,
     "start_time": "2022-11-28T03:57:06.871445",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "w={0:0.2,1:0.8}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d0d3ec0c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T03:57:06.898371Z",
     "iopub.status.busy": "2022-11-28T03:57:06.898001Z",
     "iopub.status.idle": "2022-11-28T04:04:13.005366Z",
     "shell.execute_reply": "2022-11-28T04:04:13.003526Z"
    },
    "papermill": {
     "duration": 426.117946,
     "end_time": "2022-11-28T04:04:13.009018",
     "exception": false,
     "start_time": "2022-11-28T03:57:06.891072",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression(max_iter=1000,n_jobs=-1,class_weight =w,solver='saga')\n",
    "main_full_yahoo_model=lr.fit(main_full_X_train, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "12e26a4f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T04:04:13.023787Z",
     "iopub.status.busy": "2022-11-28T04:04:13.023277Z",
     "iopub.status.idle": "2022-11-28T04:04:13.053032Z",
     "shell.execute_reply": "2022-11-28T04:04:13.051982Z"
    },
    "papermill": {
     "duration": 0.039921,
     "end_time": "2022-11-28T04:04:13.055496",
     "exception": false,
     "start_time": "2022-11-28T04:04:13.015575",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "main_full_y_pred=main_full_yahoo_model.predict(main_full_X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e063a937",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T04:04:13.070275Z",
     "iopub.status.busy": "2022-11-28T04:04:13.069406Z",
     "iopub.status.idle": "2022-11-28T04:04:13.080670Z",
     "shell.execute_reply": "2022-11-28T04:04:13.079449Z"
    },
    "papermill": {
     "duration": 0.02149,
     "end_time": "2022-11-28T04:04:13.083431",
     "exception": false,
     "start_time": "2022-11-28T04:04:13.061941",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "output_full_datfrme=pd.DataFrame({ 'qid': dt['qid'] ,'target': np.array(main_full_y_pred)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9678470d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-28T04:04:13.098072Z",
     "iopub.status.busy": "2022-11-28T04:04:13.097619Z",
     "iopub.status.idle": "2022-11-28T04:04:13.494350Z",
     "shell.execute_reply": "2022-11-28T04:04:13.493172Z"
    },
    "papermill": {
     "duration": 0.407145,
     "end_time": "2022-11-28T04:04:13.497038",
     "exception": false,
     "start_time": "2022-11-28T04:04:13.089893",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "output_full_datfrme.to_csv('count_lg_whole_15_85_saga.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61ae6d95",
   "metadata": {
    "papermill": {
     "duration": 0.005859,
     "end_time": "2022-11-28T04:04:13.509116",
     "exception": false,
     "start_time": "2022-11-28T04:04:13.503257",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 477.725594,
   "end_time": "2022-11-28T04:04:14.639953",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-11-28T03:56:16.914359",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}