5_At-risk_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier as RFC, AdaBoostClassifier as ABC\n",
    "from sklearn.neural_network import MLPClassifier as MLP\n",
    "from sklearn.linear_model import Perceptron \n",
    "from sklearn.neighbors import KNeighborsClassifier as KNN\n",
    "from sklearn.tree import DecisionTreeClassifier as DTC\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import optuna\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import OpenLA as la\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For At-risk prediction, load student vectors generated with fastText trained by ALL-2020 and Student Grades"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_data_set(dim,cl_train,cl_test):\n",
    "    Edudata = r'.\\data\\EduData_20221028'\n",
    "    train_vec_path = r'.\\data\\vectors\\norm_Student_Vctors_course{}_{}dim.csv'.format(cl_train,dim)\n",
    "    test_vec_path = r'.\\data\\vectors\\norm_Student_Vctors_course{}_{}dim.csv'.format(cl_test,dim)\n",
    "    train_vec = pd.read_csv(train_vec_path).set_index('userid')\n",
    "    test_vec = pd.read_csv(test_vec_path).set_index('userid')\n",
    "    train_grade =  la.CourseInformation(files_dir=Edudata, course_id=cl_train).grade_point_df().set_index(\"userid\")\n",
    "    test_grade = la.CourseInformation(files_dir=Edudata, course_id=cl_test).grade_point_df().set_index(\"userid\")\n",
    "    grade_dict = {'A':0, 'B':0, 'C':1, 'D':1, 'F':1}\n",
    "    train_grade =  train_grade.replace(grade_dict)\n",
    "    test_grade = test_grade.replace(grade_dict)\n",
    "    train_data = pd.merge(train_vec,train_grade,left_index=True,right_index=True,how=\"inner\")\n",
    "    test_data = pd.merge(test_vec,test_grade,left_index=True,right_index=True,how=\"inner\")\n",
    "    x_train = train_data.drop(columns=\"grade\")\n",
    "    y_train = train_data[\"grade\"]\n",
    "    x_test = test_data.drop(columns=\"grade\")\n",
    "    y_test = test_data[\"grade\"]\n",
    "    return x_train,y_train,x_test,y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def at_risk_prediction(train_data, train_label, test_data, test_label, model_k=\"rfc\"):\n",
    "    if model_k == \"rfc\":\n",
    "        model = RFC(random_state=42)\n",
    "        model_params ={\n",
    "            'n_estimators':[10,20,30,50,100],\n",
    "            'criterion': ['gini','entropy','log_loss'],\n",
    "            'max_depth' : [10,20,None]\n",
    "        }\n",
    "        model_nt = RFC(random_state=42)\n",
    "    elif model_k == \"svc\":\n",
    "        model = SVC(random_state=42)\n",
    "        model_params = {\n",
    "            \"C\": [10** i for i in range(-3,3)],\n",
    "            \"kernel\": [\"linear\",\"poly\",\"rbf\",\"sigmoid\"]\n",
    "        } \n",
    "        model_nt = SVC(random_state=42)\n",
    "    elif model_k == \"ada\":\n",
    "        model = ABC(random_state=42)\n",
    "        model_params = {\n",
    "            'n_estimators':[1,10,20,30,50,100],\n",
    "            'learning_rate':[0.1,0.5,1.0,2.0,5.0],\n",
    "            'algorithm': ['SAMME','SAMME.R']\n",
    "        }\n",
    "        model_nt = ABC(random_state=42)\n",
    "    elif model_k == 'knn':\n",
    "        model = KNN()\n",
    "        model_params = {\n",
    "            'n_neighbors' : [3,5,7,10],\n",
    "            'weights': [\"uniform\",\"distance\"],\n",
    "            'algorithm':['auto','ball_tree','kd_tree','brute']\n",
    "        }\n",
    "        model_nt = KNN()\n",
    "    #elif model_k == \n",
    "    # grid search によって最適なモデルを選択　パラメータの評価は3CV\n",
    "    gscv = GridSearchCV(model, model_params,cv=3,scoring='f1',verbose=0)\n",
    "    gscv.fit(train_data,train_label)\n",
    "    # 最もよかったパラメータ\n",
    "    best = gscv.best_estimator_\n",
    "\n",
    "    # predict with best parameters model\n",
    "    pred = best.predict(test_data)\n",
    "    f1_best = f1_score(pred,test_label)\n",
    "\n",
    "    #print(gscv.best_params_)\n",
    "    #print(gscv.best_score_)\n",
    "    #print(confusion_matrix(pred,test_label))\n",
    "\n",
    "    #\"Non Tuning model\"\n",
    "    model_nt.fit(train_data,train_label)\n",
    "    pred_nt = model_nt.predict(test_data)\n",
    "    f1_nt = f1_score(pred_nt,test_label)\n",
    "    #print(confusion_matrix(pred_nt,test_label))\n",
    "    return gscv.best_params_, f1_best, f1_nt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The result of E2Vec k100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cl_courses = [\"D-2022\",\"D-2021\",\"A-2022\",\"A-2021\"]\n",
    "dim=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\"c_train\",\"c_test\",\"dim\",\"epoch\",\"model\",\"best_param\",\"f1_bestparam\",\"f1_default\"]\n",
    "results_df = pd.DataFrame(columns=columns)\n",
    "\n",
    "for cl_test in cl_courses:\n",
    "        for cl_train in cl_courses:\n",
    "            if cl_test == cl_train:\n",
    "                continue\n",
    "            x_train,y_train,x_test,y_test = make_data_set(dim,cl_train,cl_test)\n",
    "            for model_select in [\"rfc\",\"svc\",\"ada\",'knn']:\n",
    "                 best_param, best_param_f1, nt_f1 = at_risk_prediction(x_train,y_train,x_test,y_test,model_k=model_select)\n",
    "                 row = pd.DataFrame(np.array([cl_train,cl_test,100,30,model_select,best_param,best_param_f1,nt_f1]).reshape(1,8),columns=columns)\n",
    "                 results_df = pd.concat([results_df,row],ignore_index=True)\n",
    "results_df[\"result\"] = results_df[[\"f1_bestparam\",\"f1_default\"]].max(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[(results_df[\"c_train\"]==\"A-2021\")&(results_df[\"c_test\"]==\"A-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[(results_df[\"c_train\"]==\"D-2021\")&(results_df[\"c_test\"]==\"D-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[(results_df[\"model\"]==\"rfc\")].sort_values([\"c_train\",\"c_test\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The result of E2Vec k10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cl_courses = [\"D-2022\",\"D-2021\",\"A-2022\",\"A-2021\"]\n",
    "dim=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\"c_train\",\"c_test\",\"dim\",\"epoch\",\"model\",\"best_param\",\"f1_bestparam\",\"f1_default\"]\n",
    "results_k10_df = pd.DataFrame(columns=columns)\n",
    "\n",
    "for cl_test in cl_courses:\n",
    "        for cl_train in cl_courses:\n",
    "            if cl_test == cl_train:\n",
    "                continue\n",
    "            x_train,y_train,x_test,y_test = make_data_set(10,cl_train,cl_test)\n",
    "            for model_select in [\"rfc\",\"svc\",\"ada\",'knn']:\n",
    "                 best_param, best_param_f1, nt_f1 = at_risk_prediction(x_train,y_train,x_test,y_test,model_k=model_select)\n",
    "                 row = pd.DataFrame(np.array([cl_train,cl_test,10,30,model_select,best_param,best_param_f1,nt_f1]).reshape(1,8),columns=columns)\n",
    "                 results_k10_df = pd.concat([results_k10_df,row],ignore_index=True)\n",
    "results_k10_df[\"result\"] = results_k10_df[[\"f1_bestparam\",\"f1_default\"]].max(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_k10_df[(results_k10_df[\"c_train\"]==\"A-2021\")&(results_k10_df[\"c_test\"]==\"A-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_k10_df[(results_k10_df[\"c_train\"]==\"D-2021\")&(results_k10_df[\"c_test\"]==\"D-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_k10_df[(results_k10_df[\"model\"]==\"rfc\")].sort_values([\"c_train\",\"c_test\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### E2VecA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dim =100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_data_set_A(dim,cl_train,cl_test):\n",
    "    Edudata = r'.\\data\\EduData_20221028'\n",
    "    train_vec_path = r'.\\data\\vectors\\norm_Student_Vctors_course{}_{}dim_A20.csv'.format(cl_train,dim)\n",
    "    test_vec_path = r'.\\data\\vectors\\norm_Student_Vctors_course{}_{}dim_A20.csv'.format(cl_test,dim)\n",
    "    train_vec = pd.read_csv(train_vec_path).set_index('userid')\n",
    "    test_vec = pd.read_csv(test_vec_path).set_index('userid')\n",
    "    train_grade =  la.CourseInformation(files_dir=Edudata, course_id=cl_train).grade_point_df().set_index(\"userid\")\n",
    "    test_grade = la.CourseInformation(files_dir=Edudata, course_id=cl_test).grade_point_df().set_index(\"userid\")\n",
    "    grade_dict = {'S':0, 'A':0, 'B':0, 'C':1, 'D':1, 'F':1}\n",
    "    train_grade =  train_grade.replace(grade_dict)\n",
    "    test_grade = test_grade.replace(grade_dict)\n",
    "    train_data = pd.merge(train_vec,train_grade,left_index=True,right_index=True,how=\"inner\")\n",
    "    test_data = pd.merge(test_vec,test_grade,left_index=True,right_index=True,how=\"inner\")\n",
    "    x_train = train_data.drop(columns=\"grade\")\n",
    "    y_train = train_data[\"grade\"]\n",
    "    x_test = test_data.drop(columns=\"grade\")\n",
    "    y_test = test_data[\"grade\"]\n",
    "    return x_train,y_train,x_test,y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\"c_train\",\"c_test\",\"dim\",\"epoch\",\"model\",\"best_param\",\"f1_bestparam\",\"f1_default\"]\n",
    "results_A_df = pd.DataFrame(columns=columns)\n",
    "\n",
    "for cl_test in cl_courses:\n",
    "        for cl_train in cl_courses:\n",
    "            if cl_test == cl_train:\n",
    "                continue\n",
    "            x_train,y_train,x_test,y_test = make_data_set_A(dim,cl_train,cl_test)\n",
    "            for model_select in [\"rfc\",\"svc\",\"ada\",'knn']:\n",
    "                 best_param, best_param_f1, nt_f1 = at_risk_prediction(x_train,y_train,x_test,y_test,model_k=model_select)\n",
    "                 row = pd.DataFrame(np.array([cl_train,cl_test,100,30,model_select,best_param,best_param_f1,nt_f1]).reshape(1,8),columns=columns)\n",
    "                 results_A_df = pd.concat([results_A_df,row],ignore_index=True)\n",
    "results_A_df[\"result\"] = results_A_df[[\"f1_bestparam\",\"f1_default\"]].max(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_A_df[(results_A_df[\"c_train\"]==\"A-2021\")&(results_A_df[\"c_test\"]==\"A-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_A_df[(results_A_df[\"c_train\"]==\"D-2021\")&(results_A_df[\"c_test\"]==\"D-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_A_df[(results_A_df[\"model\"]==\"rfc\")].sort_values([\"c_train\",\"c_test\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### E2VecD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_data_set_D(dim,cl_train,cl_test):\n",
    "    Edudata = r'.\\data\\EduData_20221028'\n",
    "    train_vec_path = r'.\\data\\vectors\\norm_Student_Vctors_course{}_{}dim_D20.csv'.format(cl_train,dim)\n",
    "    test_vec_path = r'.\\data\\vectors\\norm_Student_Vctors_course{}_{}dim_D20.csv'.format(cl_test,dim)\n",
    "    train_vec = pd.read_csv(train_vec_path).set_index('userid')\n",
    "    test_vec = pd.read_csv(test_vec_path).set_index('userid')\n",
    "    train_grade =  la.CourseInformation(files_dir=Edudata, course_id=cl_train).grade_point_df().set_index(\"userid\")\n",
    "    test_grade = la.CourseInformation(files_dir=Edudata, course_id=cl_test).grade_point_df().set_index(\"userid\")\n",
    "    grade_dict = {'S':0, 'A':0, 'B':0, 'C':1, 'D':1, 'F':1}\n",
    "    train_grade =  train_grade.replace(grade_dict)\n",
    "    test_grade = test_grade.replace(grade_dict)\n",
    "    train_data = pd.merge(train_vec,train_grade,left_index=True,right_index=True,how=\"inner\")\n",
    "    test_data = pd.merge(test_vec,test_grade,left_index=True,right_index=True,how=\"inner\")\n",
    "    x_train = train_data.drop(columns=\"grade\")\n",
    "    y_train = train_data[\"grade\"]\n",
    "    x_test = test_data.drop(columns=\"grade\")\n",
    "    y_test = test_data[\"grade\"]\n",
    "    return x_train,y_train,x_test,y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\"c_train\",\"c_test\",\"dim\",\"epoch\",\"model\",\"best_param\",\"f1_bestparam\",\"f1_default\"]\n",
    "results_D_df = pd.DataFrame(columns=columns)\n",
    "\n",
    "for cl_test in cl_courses:\n",
    "        for cl_train in cl_courses:\n",
    "            if cl_test == cl_train:\n",
    "                continue\n",
    "            x_train,y_train,x_test,y_test = make_data_set_D(dim,cl_train,cl_test)\n",
    "            for model_select in [\"rfc\",\"svc\",\"ada\",'knn']:\n",
    "                 best_param, best_param_f1, nt_f1 = at_risk_prediction(x_train,y_train,x_test,y_test,model_k=model_select)\n",
    "                 row = pd.DataFrame(np.array([cl_train,cl_test,100,30,model_select,best_param,best_param_f1,nt_f1]).reshape(1,8),columns=columns)\n",
    "                 results_D_df = pd.concat([results_D_df,row],ignore_index=True)\n",
    "results_D_df[\"result\"] = results_D_df[[\"f1_bestparam\",\"f1_default\"]].max(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_D_df[(results_D_df[\"c_train\"]==\"A-2021\")&(results_D_df[\"c_test\"]==\"A-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_D_df[(results_D_df[\"c_train\"]==\"D-2021\")&(results_D_df[\"c_test\"]==\"D-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_D_df[(results_D_df[\"model\"]==\"rfc\")].sort_values([\"c_train\",\"c_test\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### operation_count (OC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_oc(course_id):\n",
    "    Edudata = r'.\\data\\EduData_20221028'\n",
    "    course_info = la.CourseInformation(files_dir=Edudata , course_id=course_id)\n",
    "    event_stream = course_info.load_eventstream()\n",
    "    operation_count = la.convert_into_operation_count(event_stream=event_stream,separate_marker_type=False)\n",
    "    feature_df = operation_count.df.drop(columns='contentsid')\n",
    "    feature_df = feature_df.groupby('userid').sum()\n",
    "    feature_df = feature_df[['NEXT', 'PREV','ADD MARKER','OPEN','CLOSE','PAGE_JUMP','GETIT']]\n",
    "    norm_oc_df = pd.DataFrame(normalize(feature_df,norm=\"l2\",axis=1),index=feature_df.index)\n",
    "    #print(norm_oc_df.head())\n",
    "    return norm_oc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_data_set_oc(dim,cl_train,cl_test):\n",
    "    Edudata = r'.\\data\\EduData_20221028'\n",
    "    train_vec = make_oc(cl_train)\n",
    "    test_vec = make_oc(cl_test)\n",
    "    train_grade =  la.CourseInformation(files_dir=Edudata, course_id=cl_train).grade_point_df().set_index(\"userid\")\n",
    "    test_grade = la.CourseInformation(files_dir=Edudata, course_id=cl_test).grade_point_df().set_index(\"userid\")\n",
    "    grade_dict = {'S':0, 'A':0, 'B':0, 'C':1, 'D':1, 'F':1}\n",
    "    train_grade =  train_grade.replace(grade_dict)\n",
    "    test_grade = test_grade.replace(grade_dict)\n",
    "    train_data = pd.merge(train_vec,train_grade,left_index=True,right_index=True,how=\"inner\")\n",
    "    test_data = pd.merge(test_vec,test_grade,left_index=True,right_index=True,how=\"inner\")\n",
    "    x_train = train_data.drop(columns=\"grade\")\n",
    "    y_train = train_data[\"grade\"]\n",
    "    x_test = test_data.drop(columns=\"grade\")\n",
    "    y_test = test_data[\"grade\"]\n",
    "    return x_train,y_train,x_test,y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\"c_train\",\"c_test\",\"dim\",\"epoch\",\"model\",\"best_param\",\"f1_bestparam\",\"f1_default\"]\n",
    "results_oc_df = pd.DataFrame(columns=columns)\n",
    "\n",
    "for cl_test in cl_courses:\n",
    "        for cl_train in cl_courses:\n",
    "            if cl_test == cl_train:\n",
    "                continue\n",
    "            x_train,y_train,x_test,y_test = make_data_set_oc(dim,cl_train,cl_test)\n",
    "            for model_select in [\"rfc\",\"svc\",\"ada\",'knn']:\n",
    "                 best_param, best_param_f1, nt_f1 = at_risk_prediction(x_train,y_train,x_test,y_test,model_k=model_select)\n",
    "                 row = pd.DataFrame(np.array([cl_train,cl_test,100,30,model_select,best_param,best_param_f1,nt_f1]).reshape(1,8),columns=columns)\n",
    "                 results_oc_df = pd.concat([results_oc_df,row],ignore_index=True)\n",
    "results_oc_df[\"result\"] = results_oc_df[[\"f1_bestparam\",\"f1_default\"]].max(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_oc_df[(results_oc_df[\"c_train\"]==\"A-2021\")&(results_oc_df[\"c_test\"]==\"A-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_oc_df[(results_oc_df[\"c_train\"]==\"D-2021\")&(results_oc_df[\"c_test\"]==\"D-2022\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_oc_df[(results_oc_df[\"model\"]==\"rfc\")].sort_values([\"c_train\",\"c_test\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}