script.ipynb

{"cells":[
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "# Kobe Shots - Show Me Your Best Model\n\nThe following notebook presents a thought process of creating and debugging ML algorithm for predicting whether a shot is successfull or missed (binary classification problem).\n\n## 1. Preparation\n\n### Load libraries\nLoad all required libraries"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "%matplotlib inline \n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.decomposition import PCA, KernelPCA\nfrom sklearn.cross_validation import KFold, cross_val_score\nfrom sklearn.metrics import make_scorer\nfrom sklearn.grid_search import GridSearchCV\nfrom sklearn.feature_selection import VarianceThreshold, RFE, SelectKBest, chi2\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.pipeline import Pipeline, FeatureUnion\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import BaggingClassifier, ExtraTreesClassifier, GradientBoostingClassifier, VotingClassifier, RandomForestClassifier, AdaBoostClassifier\n\nsns.set_style('whitegrid')\npd.set_option('display.max_columns', None) # display all columns"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Load dataset\nLet's read the data from CSV file, explicity set an index and convert some columns to `category` type (for better summarization)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data = pd.read_csv('../input/data.csv')\n\ndata.set_index('shot_id', inplace=True)\ndata[\"action_type\"] = data[\"action_type\"].astype('object')\ndata[\"combined_shot_type\"] = data[\"combined_shot_type\"].astype('category')\ndata[\"game_event_id\"] = data[\"game_event_id\"].astype('category')\ndata[\"game_id\"] = data[\"game_id\"].astype('category')\ndata[\"period\"] = data[\"period\"].astype('object')\ndata[\"playoffs\"] = data[\"playoffs\"].astype('category')\ndata[\"season\"] = data[\"season\"].astype('category')\ndata[\"shot_made_flag\"] = data[\"shot_made_flag\"].astype('category')\ndata[\"shot_type\"] = data[\"shot_type\"].astype('category')\ndata[\"team_id\"] = data[\"team_id\"].astype('category')"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Quick look:"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data.head(2)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data.dtypes"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## 2. Summarize data\n\n### Descriptive statistics\nThe inital dimension of the dataset:"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data.shape"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Let's take a brief look at all numerical columns statistcs:"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data.describe(include=['number'])"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "And for categorical columns:"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data.describe(include=['object', 'category'])"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Data Visualization \nSee target class distribution"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "ax = plt.axes()\nsns.countplot(x='shot_made_flag', data=data, ax=ax);\nax.set_title('Target class distribution')\nplt.show()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "At first we can see that the target variable is distributed quite equally. We won't perform any actions to deal with imbalanced dataset.\n\nData will be presented using boxplot (described in the following image)\n\n![boxplot](https://support.sas.com/documentation/cdl/en/statug/63033/HTML/default/images/schematic.png)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "f, axarr = plt.subplots(4, 2, figsize=(15, 15))\n\nsns.boxplot(x='lat', y='shot_made_flag', data=data, showmeans=True, ax=axarr[0,0])\nsns.boxplot(x='lon', y='shot_made_flag', data=data, showmeans=True, ax=axarr[0, 1])\nsns.boxplot(x='loc_y', y='shot_made_flag', data=data, showmeans=True, ax=axarr[1, 0])\nsns.boxplot(x='loc_x', y='shot_made_flag', data=data, showmeans=True, ax=axarr[1, 1])\nsns.boxplot(x='minutes_remaining', y='shot_made_flag', showmeans=True, data=data, ax=axarr[2, 0])\nsns.boxplot(x='seconds_remaining', y='shot_made_flag', showmeans=True, data=data, ax=axarr[2, 1])\nsns.boxplot(x='shot_distance', y='shot_made_flag', data=data, showmeans=True, ax=axarr[3, 0])\n\naxarr[0, 0].set_title('Latitude')\naxarr[0, 1].set_title('Longitude')\naxarr[1, 0].set_title('Loc y')\naxarr[1, 1].set_title('Loc x')\naxarr[2, 0].set_title('Minutes remaining')\naxarr[2, 1].set_title('Seconds remaining')\naxarr[3, 0].set_title('Shot distance')\n\nplt.tight_layout()\nplt.show()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "sns.pairplot(data, vars=['loc_x', 'loc_y', 'lat', 'lon', 'shot_distance'], hue='shot_made_flag', size=3)\nplt.show()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "f, axarr = plt.subplots(8, figsize=(15, 25))\n\nsns.countplot(x=\"combined_shot_type\", hue=\"shot_made_flag\", data=data, ax=axarr[0])\nsns.countplot(x=\"season\", hue=\"shot_made_flag\", data=data, ax=axarr[1])\nsns.countplot(x=\"period\", hue=\"shot_made_flag\", data=data, ax=axarr[2])\nsns.countplot(x=\"playoffs\", hue=\"shot_made_flag\", data=data, ax=axarr[3])\nsns.countplot(x=\"shot_type\", hue=\"shot_made_flag\", data=data, ax=axarr[4])\nsns.countplot(x=\"shot_zone_area\", hue=\"shot_made_flag\", data=data, ax=axarr[5])\nsns.countplot(x=\"shot_zone_basic\", hue=\"shot_made_flag\", data=data, ax=axarr[6])\nsns.countplot(x=\"shot_zone_range\", hue=\"shot_made_flag\", data=data, ax=axarr[7])\n\naxarr[0].set_title('Combined shot type')\naxarr[1].set_title('Season')\naxarr[2].set_title('Period')\naxarr[3].set_title('Playoffs')\naxarr[4].set_title('Shot Type')\naxarr[5].set_title('Shot Zone Area')\naxarr[6].set_title('Shot Zone Basic')\naxarr[7].set_title('Shot Zone Range')\n\nplt.tight_layout()\nplt.show()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## 3. Prepare Data"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "unknown_mask = data['shot_made_flag'].isnull()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Data Cleaning\nWe are assuming an independence of each shot - therefore some columns might be dropped"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data_cl = data.copy() # create a copy of data frame\ntarget = data_cl['shot_made_flag'].copy()\n\n# Remove some columns\ndata_cl.drop('team_id', axis=1, inplace=True) # Always one number\ndata_cl.drop('lat', axis=1, inplace=True) # Correlated with loc_x\ndata_cl.drop('lon', axis=1, inplace=True) # Correlated with loc_y\ndata_cl.drop('game_id', axis=1, inplace=True) # Independent\ndata_cl.drop('game_event_id', axis=1, inplace=True) # Independent\ndata_cl.drop('team_name', axis=1, inplace=True) # Always LA Lakers\ndata_cl.drop('shot_made_flag', axis=1, inplace=True)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "There are also many outliers, remove them:"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "def detect_outliers(series, whis=1.5):\n    q75, q25 = np.percentile(series, [75 ,25])\n    iqr = q75 - q25\n    return ~((series - series.median()).abs() <= (whis * iqr))\n\n## For now - do not remove anything"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Data Transformation\n\n##### New features"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Remaining time\ndata_cl['seconds_from_period_end'] = 60 * data_cl['minutes_remaining'] + data_cl['seconds_remaining']\ndata_cl['last_5_sec_in_period'] = data_cl['seconds_from_period_end'] < 5\n\ndata_cl.drop('minutes_remaining', axis=1, inplace=True)\ndata_cl.drop('seconds_remaining', axis=1, inplace=True)\ndata_cl.drop('seconds_from_period_end', axis=1, inplace=True)\n\n## Matchup - (away/home)\ndata_cl['home_play'] = data_cl['matchup'].str.contains('vs').astype('int')\ndata_cl.drop('matchup', axis=1, inplace=True)\n\n# Game date\ndata_cl['game_date'] = pd.to_datetime(data_cl['game_date'])\ndata_cl['game_year'] = data_cl['game_date'].dt.year\ndata_cl['game_month'] = data_cl['game_date'].dt.month\ndata_cl.drop('game_date', axis=1, inplace=True)\n\n# Loc_x, and loc_y binning\ndata_cl['loc_x'] = pd.cut(data_cl['loc_x'], 25)\ndata_cl['loc_y'] = pd.cut(data_cl['loc_y'], 25)\n\n# Replace 20 least common action types with value 'Other'\nrare_action_types = data_cl['action_type'].value_counts().sort_values().index.values[:20]\ndata_cl.loc[data_cl['action_type'].isin(rare_action_types), 'action_type'] = 'Other'"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "##### Encode categorical variables"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "categorial_cols = [\n    'action_type', 'combined_shot_type', 'period', 'season', 'shot_type',\n    'shot_zone_area', 'shot_zone_basic', 'shot_zone_range', 'game_year',\n    'game_month', 'opponent', 'loc_x', 'loc_y']\n\nfor cc in categorial_cols:\n    dummies = pd.get_dummies(data_cl[cc])\n    dummies = dummies.add_prefix(\"{}#\".format(cc))\n    data_cl.drop(cc, axis=1, inplace=True)\n    data_cl = data_cl.join(dummies)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Maybe some transformations to Gaussian distribution?"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# TODO"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## Feature Selection\nLet's reduce the number of features\n\nCreate views for easier analysis"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Separate dataset for validation\ndata_submit = data_cl[unknown_mask]\n\n# Separate dataset for training\nX = data_cl[~unknown_mask]\nY = target[~unknown_mask]"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Variance Threshold\nFind all features with more than 90% variance in values."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "threshold = 0.90\nvt = VarianceThreshold().fit(X)\n\n# Find feature names\nfeat_var_threshold = data_cl.columns[vt.variances_ > threshold * (1-threshold)]\nfeat_var_threshold"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Top 20 most important features\nAccording to `RandomForestClassifier`"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "model = RandomForestClassifier()\nmodel.fit(X, Y)\n\nfeature_imp = pd.DataFrame(model.feature_importances_, index=X.columns, columns=[\"importance\"])\nfeat_imp_20 = feature_imp.sort_values(\"importance\", ascending=False).head(20).index\nfeat_imp_20"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Univariate feature selection\nSelect top 20 features using $chi^2$ test. Features must be positive before applying test."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "X_minmax = MinMaxScaler(feature_range=(0,1)).fit_transform(X)\nX_scored = SelectKBest(score_func=chi2, k='all').fit(X_minmax, Y)\nfeature_scoring = pd.DataFrame({\n        'feature': X.columns,\n        'score': X_scored.scores_\n    })\n\nfeat_scored_20 = feature_scoring.sort_values('score', ascending=False).head(20)['feature'].values\nfeat_scored_20"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Recursive Feature Elimination\nSelect 20 features from using recursive feature elimination (RFE) with logistic regression model."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "rfe = RFE(LogisticRegression(), 20)\nrfe.fit(X, Y)\n\nfeature_rfe_scoring = pd.DataFrame({\n        'feature': X.columns,\n        'score': rfe.ranking_\n    })\n\nfeat_rfe_20 = feature_rfe_scoring[feature_rfe_scoring['score'] == 1]['feature'].values\nfeat_rfe_20"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "# Final feature selection\nFinally features selected by all methods will be merged together"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "features = np.hstack([\n        feat_var_threshold, \n        feat_imp_20,\n        feat_scored_20,\n        feat_rfe_20\n    ])\n\nfeatures = np.unique(features)\nprint('Final features set:\\n')\nfor f in features:\n    print(\"\\t-{}\".format(f))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## Prepare dataset for further analysis"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "data_cl = data_cl.ix[:, features]\ndata_submit = data_submit.ix[:, features]\nX = X.ix[:, features]\n\nprint('Clean dataset shape: {}'.format(data_cl.shape))\nprint('Subbmitable dataset shape: {}'.format(data_submit.shape))\nprint('Train features shape: {}'.format(X.shape))\nprint('Target label shape: {}'. format(Y.shape))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## PCA Visualization"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "components = 8\npca = PCA(n_components=components).fit(X)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Show explained variance for each component"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "pca_variance_explained_df = pd.DataFrame({\n    \"component\": np.arange(1, components+1),\n    \"variance_explained\": pca.explained_variance_ratio_            \n    })\n\nax = sns.barplot(x='component', y='variance_explained', data=pca_variance_explained_df)\nax.set_title(\"PCA - Variance explained\")\nplt.show()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "X_pca = pd.DataFrame(pca.transform(X)[:,:2])\nX_pca['target'] = Y.values\nX_pca.columns = [\"x\", \"y\", \"target\"]\n\nsns.lmplot('x','y', \n           data=X_pca, \n           hue=\"target\", \n           fit_reg=False, \n           markers=[\"o\", \"x\"], \n           palette=\"Set1\", \n           size=7,\n           scatter_kws={\"alpha\": .2}\n          )\nplt.show()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## 4. Evaluate Algorithms"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "seed = 7\nprocessors=1\nnum_folds=3\nnum_instances=len(X)\nscoring='log_loss'\n\nkfold = KFold(n=num_instances, n_folds=num_folds, random_state=seed)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## Algorithms spot-check"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Prepare some basic models\nmodels = []\nmodels.append(('LR', LogisticRegression()))\nmodels.append(('LDA', LinearDiscriminantAnalysis()))\nmodels.append(('K-NN', KNeighborsClassifier(n_neighbors=5)))\nmodels.append(('CART', DecisionTreeClassifier()))\nmodels.append(('NB', GaussianNB()))\n#models.append(('SVC', SVC(probability=True)))\n\n# Evaluate each model in turn\nresults = []\nnames = []\n\nfor name, model in models:\n    cv_results = cross_val_score(model, X, Y, cv=kfold, scoring=scoring, n_jobs=processors)\n    results.append(cv_results)\n    names.append(name)\n    print(\"{0}: ({1:.3f}) +/- ({2:.3f})\".format(name, cv_results.mean(), cv_results.std()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "By looking at these results is seems that only Logistic Regression and Linear Discriminant Analysis are providing best results and are worth further examination.\n\nBut let's look at ...\n\n## Ensembles\n\n### Bagging (Bootstrap Aggregation)\nInvolves taking multiple samples from the training dataset (with replacement) and training a model for each sample. The final output prediction is averaged across the predictions of all of the sub-models.\n\n#### Bagged Decision Trees\nBagging performs best with algorithms that have high variance (i.e. decision trees without prunning). Let's check their performance"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "cart = DecisionTreeClassifier()\nnum_trees = 100\n\nmodel = BaggingClassifier(base_estimator=cart, n_estimators=num_trees, random_state=seed)\n\nresults = cross_val_score(model, X, Y, cv=kfold, scoring=scoring, n_jobs=processors)\nprint(\"({0:.3f}) +/- ({1:.3f})\".format(results.mean(), results.std()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "#### Random Forest\nAn extension to bagged decision trees. Samples of the training dataset are taken with replacement, but the trees are constructed in a way that reduces the correlation between individual classifiers. Also the tree size is much slowe due to `max_features`"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "num_trees = 100\nnum_features = 10\n\nmodel = RandomForestClassifier(n_estimators=num_trees, max_features=num_features)\n\nresults = cross_val_score(model, X, Y, cv=kfold, scoring=scoring, n_jobs=processors)\nprint(\"({0:.3f}) +/- ({1:.3f})\".format(results.mean(), results.std()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "#### Extra Trees\nIn extremely randomized trees, randomness goes one step further in the way splits are computed. As in random forests, a random subset of candidate features is used, but instead of looking for the most discriminative thresholds, thresholds are drawn at random for each candidate feature and the best of these randomly-generated thresholds is picked as the splitting rule. This usually allows to reduce the variance of the model a bit more, at the expense of a slightly greater increase in bias"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "num_trees = 100\nnum_features = 10\n\nmodel = ExtraTreesClassifier(n_estimators=num_trees, max_features=num_features)\n\nresults = cross_val_score(model, X, Y, cv=kfold, scoring=scoring, n_jobs=processors)\nprint(\"({0:.3f}) +/- ({1:.3f})\".format(results.mean(), results.std()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Boosting\nBoosting ensembles creates a sequence of models that attemtp to correct the mistakes of the models before them in the sequence. Once created, the models make predictions which may be weighted by their demonstrated accuracy and the results are combined to create a final output prediction.\n\n#### AdaBoost\n\nThe core principle of AdaBoost is to fit a sequence of weak learners (i.e., models that are only slightly better than random guessing, such as small decision trees) on repeatedly modified versions of the data. The predictions from all of them are then combined through a weighted majority vote (or sum) to produce the final prediction. The data modifications at each so-called boosting iteration consist of applying weights $w_1, w_2, ..., w_N$ to each of the training samples. Initially, those weights are all set to $w_i = 1/N$, so that the first step simply trains a weak learner on the original data. For each successive iteration, the sample weights are individually modified and the learning algorithm is reapplied to the reweighted data. At a given step, those training examples that were incorrectly predicted by the boosted model induced at the previous step have their weights increased, whereas the weights are decreased for those that were predicted correctly. As iterations proceed, examples that are difficult to predict receive ever-increasing influence. Each subsequent weak learner is thereby forced to concentrate on the examples that are missed by the previous ones in the sequence"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "model = AdaBoostClassifier(n_estimators=100, random_state=seed)\n\nresults = cross_val_score(model, X, Y, cv=kfold, scoring=scoring, n_jobs=processors)\nprint(\"({0:.3f}) +/- ({1:.3f})\".format(results.mean(), results.std()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "#### Stochastic Gradient Boosting\nGradient Tree Boosting or Gradient Boosted Regression Trees (GBRT) is a generalization of boosting to arbitrary differentiable loss functions. GBRT is an accurate and effective off-the-shelf procedure that can be used for both regression and classification problems.\n\nThe advantages of GBRT are:\n\n- Natural handling of data of mixed type (= heterogeneous features)\n- Predictive power\n- Robustness to outliers in output space (via robust loss functions)\n\nThe disadvantages of GBRT are:\n\n- Scalability, due to the sequential nature of boosting it can hardly be parallelized."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "model = GradientBoostingClassifier(n_estimators=100, random_state=seed)\n\nresults = cross_val_score(model, X, Y, cv=kfold, scoring=scoring, n_jobs=processors)\nprint(\"({0:.3f}) +/- ({1:.3f})\".format(results.mean(), results.std()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Hyperparameter tuning\n#### Logistic Regression"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "lr_grid = GridSearchCV(\n    estimator = LogisticRegression(random_state=seed),\n    param_grid = {\n        'penalty': ['l1', 'l2'],\n        'C': [0.001, 0.01, 1, 10, 100, 1000]\n    }, \n    cv = kfold, \n    scoring = scoring, \n    n_jobs = processors)\n\nlr_grid.fit(X, Y)\n\nprint(lr_grid.best_score_)\nprint(lr_grid.best_params_)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "#### Linear Discriminant Analysis"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "lda_grid = GridSearchCV(\n    estimator = LinearDiscriminantAnalysis(),\n    param_grid = {\n        'solver': ['lsqr'],\n        'shrinkage': [0, 0.25, 0.5, 0.75, 1],\n        'n_components': [None, 2, 5, 10]\n    }, \n    cv = kfold, \n    scoring = scoring, \n    n_jobs = processors)\n\nlda_grid.fit(X, Y)\n\nprint(lda_grid.best_score_)\nprint(lda_grid.best_params_)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### K-NN"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "knn_grid = GridSearchCV(\n    estimator = Pipeline([\n        ('min_max_scaler', MinMaxScaler()),\n        ('knn', KNeighborsClassifier())\n    ]),\n    param_grid = {\n        'knn__n_neighbors': [25],\n        'knn__algorithm': ['ball_tree'],\n        'knn__leaf_size': [2, 3, 4],\n        'knn__p': [1]\n    }, \n    cv = kfold, \n    scoring = scoring, \n    n_jobs = processors)\n\nknn_grid.fit(X, Y)\n\nprint(knn_grid.best_score_)\nprint(knn_grid.best_params_)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "#### Random Forest"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "rf_grid = GridSearchCV(\n    estimator = RandomForestClassifier(warm_start=True, random_state=seed),\n    param_grid = {\n        'n_estimators': [100, 200],\n        'criterion': ['gini', 'entropy'],\n        'max_features': [18, 20],\n        'max_depth': [8, 10],\n        'bootstrap': [True]\n    }, \n    cv = kfold, \n    scoring = scoring, \n    n_jobs = processors)\n\nrf_grid.fit(X, Y)\n\nprint(rf_grid.best_score_)\nprint(rf_grid.best_params_)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "#### AdaBoost"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "ada_grid = GridSearchCV(\n    estimator = AdaBoostClassifier(random_state=seed),\n    param_grid = {\n        'algorithm': ['SAMME', 'SAMME.R'],\n        'n_estimators': [10, 25, 50],\n        'learning_rate': [1e-3, 1e-2, 1e-1]\n    }, \n    cv = kfold, \n    scoring = scoring, \n    n_jobs = processors)\n\nada_grid.fit(X, Y)\n\nprint(ada_grid.best_score_)\nprint(ada_grid.best_params_)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "#### Gradient Boosting"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "gbm_grid = GridSearchCV(\n    estimator = GradientBoostingClassifier(warm_start=True, random_state=seed),\n    param_grid = {\n        'n_estimators': [100, 200],\n        'max_depth': [2, 3, 4],\n        'max_features': [10, 15, 20],\n        'learning_rate': [1e-1, 1]\n    }, \n    cv = kfold, \n    scoring = scoring, \n    n_jobs = processors)\n\ngbm_grid.fit(X, Y)\n\nprint(gbm_grid.best_score_)\nprint(gbm_grid.best_params_)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### Voting ensemble"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Create sub models\nestimators = []\n\nestimators.append(('lr', LogisticRegression(penalty='l2', C=1)))\nestimators.append(('gbm', GradientBoostingClassifier(n_estimators=200, max_depth=3, learning_rate=0.1, max_features=15, warm_start=True, random_state=seed)))\nestimators.append(('rf', RandomForestClassifier(bootstrap=True, max_depth=8, n_estimators=200, max_features=20, criterion='entropy', random_state=seed)))\nestimators.append(('ada', AdaBoostClassifier(algorithm='SAMME.R', learning_rate=1e-2, n_estimators=10, random_state=seed)))\n\n# create the ensemble model\nensemble = VotingClassifier(estimators, voting='soft', weights=[2,3,3,1])\n\nresults = cross_val_score(ensemble, X, Y, cv=kfold, scoring=scoring,n_jobs=processors)\nprint(\"({0:.3f}) +/- ({1:.3f})\".format(results.mean(), results.std()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## Make final predictions"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "model = ensemble\n\nmodel.fit(X, Y)\npreds = model.predict_proba(data_submit)\n\nsubmission = pd.DataFrame()\nsubmission[\"shot_id\"] = data_submit.index\nsubmission[\"shot_made_flag\"]= preds[:,0]\n\nsubmission.to_csv(\"sub.csv\",index=False)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}