python_proj/modelling/notebooks/preprocessing/feature_construction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:19.098116Z",
     "iopub.status.busy": "2024-02-21T16:38:19.097946Z",
     "iopub.status.idle": "2024-02-21T16:38:31.873999Z",
     "shell.execute_reply": "2024-02-21T16:38:31.873617Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_path='/workspaces/msc_thesis/data/final_data/non_ftc_data.csv'\n",
      "len(df)=1815972.\n",
      "len(df.columns)=155\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Submitter ID</th>\n",
       "      <th>PR Number</th>\n",
       "      <th>ControlPullRequestLifeTimeInMinutes</th>\n",
       "      <th>ControlNumberOfCommitsInPullRequest</th>\n",
       "      <th>IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRIntegratorToSubmitterV2-In)</th>\n",
       "      <th>IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToSubmitterV2-In)</th>\n",
       "      <th>IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToCommenterV2-In)</th>\n",
       "      <th>IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToCommenterV2-In)</th>\n",
       "      <th>IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToSubmitterV2-In)</th>\n",
       "      <th>...</th>\n",
       "      <th>EcosystemSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter</th>\n",
       "      <th>EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>EcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>EcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.474751e+08</td>\n",
       "      <td>5.198276e+06</td>\n",
       "      <td>3.807207e+03</td>\n",
       "      <td>3.190111e+04</td>\n",
       "      <td>4.947501e+00</td>\n",
       "      <td>3.926438e+01</td>\n",
       "      <td>1.076084e+01</td>\n",
       "      <td>1.262702e+01</td>\n",
       "      <td>8.083162e+00</td>\n",
       "      <td>8.484018e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>8.594130e-02</td>\n",
       "      <td>1.475467e+00</td>\n",
       "      <td>2.615631e+00</td>\n",
       "      <td>1.785944e+01</td>\n",
       "      <td>1.743463e-01</td>\n",
       "      <td>8.645552e-01</td>\n",
       "      <td>2.274088e+00</td>\n",
       "      <td>1.570265e+01</td>\n",
       "      <td>1.854588e-01</td>\n",
       "      <td>1.355166e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.004294e+08</td>\n",
       "      <td>8.100370e+06</td>\n",
       "      <td>8.071615e+03</td>\n",
       "      <td>1.399335e+05</td>\n",
       "      <td>7.283362e+01</td>\n",
       "      <td>2.497234e+03</td>\n",
       "      <td>3.186353e+02</td>\n",
       "      <td>4.506868e+02</td>\n",
       "      <td>3.652262e+02</td>\n",
       "      <td>1.962613e+02</td>\n",
       "      <td>...</td>\n",
       "      <td>1.285316e+00</td>\n",
       "      <td>2.382904e+01</td>\n",
       "      <td>7.985540e+00</td>\n",
       "      <td>6.700519e+01</td>\n",
       "      <td>1.580686e+00</td>\n",
       "      <td>1.186131e+01</td>\n",
       "      <td>7.064174e+00</td>\n",
       "      <td>6.131080e+01</td>\n",
       "      <td>2.446396e+00</td>\n",
       "      <td>1.667164e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>6.870000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.666667e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.807322e+07</td>\n",
       "      <td>4.631570e+05</td>\n",
       "      <td>1.000000e+02</td>\n",
       "      <td>5.290000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.371940e+08</td>\n",
       "      <td>1.620265e+06</td>\n",
       "      <td>5.480000e+02</td>\n",
       "      <td>9.799167e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.278026e+08</td>\n",
       "      <td>6.384100e+06</td>\n",
       "      <td>2.817000e+03</td>\n",
       "      <td>7.328350e+03</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>7.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.617756e+08</td>\n",
       "      <td>5.976123e+07</td>\n",
       "      <td>8.297600e+04</td>\n",
       "      <td>4.489490e+06</td>\n",
       "      <td>1.000000e+04</td>\n",
       "      <td>3.461280e+05</td>\n",
       "      <td>8.789200e+04</td>\n",
       "      <td>1.227920e+05</td>\n",
       "      <td>1.095040e+05</td>\n",
       "      <td>4.764000e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>2.190000e+02</td>\n",
       "      <td>5.687000e+03</td>\n",
       "      <td>2.332000e+03</td>\n",
       "      <td>7.630000e+03</td>\n",
       "      <td>3.730000e+02</td>\n",
       "      <td>9.270000e+02</td>\n",
       "      <td>2.332000e+03</td>\n",
       "      <td>7.630000e+03</td>\n",
       "      <td>7.410000e+02</td>\n",
       "      <td>2.764000e+03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 147 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  Submitter ID     PR Number   \n",
       "count  1.815972e+06  1.815972e+06  1.815972e+06  \\\n",
       "mean   1.474751e+08  5.198276e+06  3.807207e+03   \n",
       "std    1.004294e+08  8.100370e+06  8.071615e+03   \n",
       "min    6.870000e+02  1.000000e+00  1.000000e+00   \n",
       "25%    5.807322e+07  4.631570e+05  1.000000e+02   \n",
       "50%    1.371940e+08  1.620265e+06  5.480000e+02   \n",
       "75%    2.278026e+08  6.384100e+06  2.817000e+03   \n",
       "max    3.617756e+08  5.976123e+07  8.297600e+04   \n",
       "\n",
       "       ControlPullRequestLifeTimeInMinutes   \n",
       "count                         1.815972e+06  \\\n",
       "mean                          3.190111e+04   \n",
       "std                           1.399335e+05   \n",
       "min                           1.666667e-02   \n",
       "25%                           5.290000e+01   \n",
       "50%                           9.799167e+02   \n",
       "75%                           7.328350e+03   \n",
       "max                           4.489490e+06   \n",
       "\n",
       "       ControlNumberOfCommitsInPullRequest   \n",
       "count                         1.815972e+06  \\\n",
       "mean                          4.947501e+00   \n",
       "std                           7.283362e+01   \n",
       "min                           0.000000e+00   \n",
       "25%                           1.000000e+00   \n",
       "50%                           1.000000e+00   \n",
       "75%                           3.000000e+00   \n",
       "max                           1.000000e+04   \n",
       "\n",
       "       IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRIntegratorToSubmitterV2-In)   \n",
       "count                                       1.815972e+06                                                  \\\n",
       "mean                                        3.926438e+01                                                   \n",
       "std                                         2.497234e+03                                                   \n",
       "min                                         0.000000e+00                                                   \n",
       "25%                                         0.000000e+00                                                   \n",
       "50%                                         0.000000e+00                                                   \n",
       "75%                                         0.000000e+00                                                   \n",
       "max                                         3.461280e+05                                                   \n",
       "\n",
       "       IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToSubmitterV2-In)   \n",
       "count                                       1.815972e+06                                                 \\\n",
       "mean                                        1.076084e+01                                                  \n",
       "std                                         3.186353e+02                                                  \n",
       "min                                         0.000000e+00                                                  \n",
       "25%                                         0.000000e+00                                                  \n",
       "50%                                         0.000000e+00                                                  \n",
       "75%                                         0.000000e+00                                                  \n",
       "max                                         8.789200e+04                                                  \n",
       "\n",
       "       IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToCommenterV2-In)   \n",
       "count                                       1.815972e+06                                                 \\\n",
       "mean                                        1.262702e+01                                                  \n",
       "std                                         4.506868e+02                                                  \n",
       "min                                         0.000000e+00                                                  \n",
       "25%                                         0.000000e+00                                                  \n",
       "50%                                         0.000000e+00                                                  \n",
       "75%                                         0.000000e+00                                                  \n",
       "max                                         1.227920e+05                                                  \n",
       "\n",
       "       IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToCommenterV2-In)   \n",
       "count                                       1.815972e+06                                                    \\\n",
       "mean                                        8.083162e+00                                                     \n",
       "std                                         3.652262e+02                                                     \n",
       "min                                         0.000000e+00                                                     \n",
       "25%                                         0.000000e+00                                                     \n",
       "50%                                         0.000000e+00                                                     \n",
       "75%                                         0.000000e+00                                                     \n",
       "max                                         1.095040e+05                                                     \n",
       "\n",
       "       IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToSubmitterV2-In)   \n",
       "count                                       1.815972e+06                                                    \\\n",
       "mean                                        8.484018e+00                                                     \n",
       "std                                         1.962613e+02                                                     \n",
       "min                                         0.000000e+00                                                     \n",
       "25%                                         0.000000e+00                                                     \n",
       "50%                                         0.000000e+00                                                     \n",
       "75%                                         0.000000e+00                                                     \n",
       "max                                         4.764000e+04                                                     \n",
       "\n",
       "       ...   \n",
       "count  ...  \\\n",
       "mean   ...   \n",
       "std    ...   \n",
       "min    ...   \n",
       "25%    ...   \n",
       "50%    ...   \n",
       "75%    ...   \n",
       "max    ...   \n",
       "\n",
       "       EcosystemSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter   \n",
       "count                                       1.815972e+06                          \\\n",
       "mean                                        8.594130e-02                           \n",
       "std                                         1.285316e+00                           \n",
       "min                                         0.000000e+00                           \n",
       "25%                                         0.000000e+00                           \n",
       "50%                                         0.000000e+00                           \n",
       "75%                                         0.000000e+00                           \n",
       "max                                         2.190000e+02                           \n",
       "\n",
       "       EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                              \\\n",
       "mean                                        1.475467e+00                               \n",
       "std                                         2.382904e+01                               \n",
       "min                                         0.000000e+00                               \n",
       "25%                                         0.000000e+00                               \n",
       "50%                                         0.000000e+00                               \n",
       "75%                                         0.000000e+00                               \n",
       "max                                         5.687000e+03                               \n",
       "\n",
       "       EcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                      1.815972e+06  \\\n",
       "mean                                       2.615631e+00   \n",
       "std                                        7.985540e+00   \n",
       "min                                        0.000000e+00   \n",
       "25%                                        0.000000e+00   \n",
       "50%                                        0.000000e+00   \n",
       "75%                                        2.000000e+00   \n",
       "max                                        2.332000e+03   \n",
       "\n",
       "       EcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                   1.815972e+06  \\\n",
       "mean                                    1.785944e+01   \n",
       "std                                     6.700519e+01   \n",
       "min                                     0.000000e+00   \n",
       "25%                                     0.000000e+00   \n",
       "50%                                     0.000000e+00   \n",
       "75%                                     7.000000e+00   \n",
       "max                                     7.630000e+03   \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.815972e+06           \\\n",
       "mean                                        1.743463e-01            \n",
       "std                                         1.580686e+00            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         0.000000e+00            \n",
       "75%                                         0.000000e+00            \n",
       "max                                         3.730000e+02            \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.815972e+06        \\\n",
       "mean                                        8.645552e-01         \n",
       "std                                         1.186131e+01         \n",
       "min                                         0.000000e+00         \n",
       "25%                                         0.000000e+00         \n",
       "50%                                         0.000000e+00         \n",
       "75%                                         0.000000e+00         \n",
       "max                                         9.270000e+02         \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.815972e+06              \\\n",
       "mean                                        2.274088e+00               \n",
       "std                                         7.064174e+00               \n",
       "min                                         0.000000e+00               \n",
       "25%                                         0.000000e+00               \n",
       "50%                                         0.000000e+00               \n",
       "75%                                         2.000000e+00               \n",
       "max                                         2.332000e+03               \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.815972e+06           \\\n",
       "mean                                        1.570265e+01            \n",
       "std                                         6.131080e+01            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         0.000000e+00            \n",
       "75%                                         6.000000e+00            \n",
       "max                                         7.630000e+03            \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.815972e+06                   \\\n",
       "mean                                        1.854588e-01                    \n",
       "std                                         2.446396e+00                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         0.000000e+00                    \n",
       "75%                                         0.000000e+00                    \n",
       "max                                         7.410000e+02                    \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueCommentCount  \n",
       "count                                       1.815972e+06                \n",
       "mean                                        1.355166e+00                \n",
       "std                                         1.667164e+01                \n",
       "min                                         0.000000e+00                \n",
       "25%                                         0.000000e+00                \n",
       "50%                                         0.000000e+00                \n",
       "75%                                         0.000000e+00                \n",
       "max                                         2.764000e+03                \n",
       "\n",
       "[8 rows x 147 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import python_proj.utils.exp_utils as exp_utils\n",
    "\n",
    "\n",
    "data_path_format = exp_utils.BASE_PATH + \"/final_data/{data_file_name}.csv\"\n",
    "\n",
    "# Details for loading non-obfuscated data.\n",
    "input_data_file_name = \"non_ftc_data\"\n",
    "data_path = data_path_format.format(data_file_name=input_data_file_name)\n",
    "print(f\"{data_path=}\")\n",
    "\n",
    "# Details obfuscated data if necessary.\n",
    "obf_input_data_file_name = \"non_ftc_data_obfuscated\"\n",
    "obf_data_path = data_path_format.format(data_file_name=obf_input_data_file_name)\n",
    "using_obfuscated_data = os.path.exists(obf_data_path)\n",
    "if using_obfuscated_data:\n",
    "    print(f\"Using obfuscated data file instead.\")\n",
    "    data_path = obf_data_path\n",
    "    print(f\"{data_path=}\")\n",
    "\n",
    "# Loads the data\n",
    "df: pd.DataFrame = pd.read_csv(filepath_or_buffer=data_path, header=0)\n",
    "\n",
    "print(f\"{len(df)=}.\")\n",
    "print(f\"{len(df.columns)=}\\n\")\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Construction\n",
    "\n",
    "This script constructs the following features:\n",
    "- Is First-time contributor field\n",
    "- Second-order degree centrality\n",
    "- Link intensity\n",
    "\n",
    "These fields are added to the dataset, removing the fields they are composed of (in case they are aggregated features).\n",
    "The final dataset is outputted to a `.csv` file."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:31.875790Z",
     "iopub.status.busy": "2024-02-21T16:38:31.875648Z",
     "iopub.status.idle": "2024-02-21T16:38:31.878195Z",
     "shell.execute_reply": "2024-02-21T16:38:31.877905Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ID', 'Project Name', 'Submitter ID', 'PR Number', 'Closed At', 'PullRequestIsMerged', 'SubmitterIsFirstTimeContributor', 'ControlIntegratedBySameUser', 'ControlPullRequestLifeTimeInMinutes', 'ControlPullRequestHasComments', 'ControlNumberOfCommitsInPullRequest', 'ControlPullRequestHasCommentByExternalUser', 'ControlHasHashTagInDescription', 'IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRIntegratorToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.PRIntegratorToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.PRCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.PRCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.IssueCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.IssueCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.PRIntegratorToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.PRCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.PRCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.IssueCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.IssueCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.PRIntegratorToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.PRCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.PRCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.IssueCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.IssueCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.PRIntegratorToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.PRCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.PRCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.IssueCommenterToCommenterV2-In)', 'IntraProjectSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.IssueCommenterToSubmitterV2-In)', 'IntraProjectSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.PRIntegratorToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.PRIntegratorToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.PRCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.PRCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.IssueCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.IssueCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.PRIntegratorToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.PRCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.PRCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.IssueCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.IssueCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.PRIntegratorToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.PRCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.PRCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.IssueCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.IssueCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.PRIntegratorToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.PRCommenterToSubmitterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.PRCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.IssueCommenterToCommenterV2-Out)', 'IntraProjectSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.IssueCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRIntegratorToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.PRIntegratorToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.PRCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.PRCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.IssueCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToSubmitterV2.IssueCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.PRIntegratorToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.PRCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.PRCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.IssueCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(PRCommenterToCommenterV2.IssueCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.PRIntegratorToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.PRCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.PRCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.IssueCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToCommenterV2.IssueCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.PRIntegratorToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.PRCommenterToSubmitterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.PRCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.IssueCommenterToCommenterV2-In)', 'EcosystemSecondOrderInDegreeCentrality(IssueCommenterToSubmitterV2.IssueCommenterToSubmitterV2-In)', 'EcosystemSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.PRIntegratorToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.PRCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRIntegratorToSubmitterV2.IssueCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.PRIntegratorToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.PRCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.PRCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.IssueCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToSubmitterV2.IssueCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.PRIntegratorToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.PRCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.PRCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.IssueCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(PRCommenterToCommenterV2.IssueCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.PRIntegratorToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.PRCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.PRCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.IssueCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToCommenterV2.IssueCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.PRIntegratorToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.PRCommenterToSubmitterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.PRCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.IssueCommenterToCommenterV2-Out)', 'EcosystemSecondOrderOutDegreeCentrality(IssueCommenterToSubmitterV2.IssueCommenterToSubmitterV2-Out)', 'ControlIntraProjectPullRequestExperienceOfIntegrator', 'IntraProjectSubmitterPullRequestSubmissionCount', 'IntraProjectSubmitterPullRequestSuccessRate', 'IntraProjectSubmitterPullRequestCommentCount', 'IntraProjectSharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator', 'IntraProjectSharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter', 'IntraProjectSharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator', 'IntraProjectSharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter', 'IntraProjectSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter', 'EcosystemSharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator', 'EcosystemSharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter', 'EcosystemSharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator', 'EcosystemSharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter', 'EcosystemSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter', 'EcosystemExperienceSubmitterPullRequestSuccessRate', 'EcosystemExperienceSubmitterPullRequestSubmissionCount', 'EcosystemExperienceSubmitterPullRequestCommentCount', 'DependencyEcosystemExperienceSubmitterPullRequestSubmissionCount', 'DependencyEcosystemExperienceSubmitterPullRequestSuccessRate', 'DependencyEcosystemExperienceSubmitterPullRequestCommentCount', 'NonDependencyEcosystemExperienceSubmitterPullRequestSubmissionCount', 'NonDependencyEcosystemExperienceSubmitterPullRequestSuccessRate', 'NonDependencyEcosystemExperienceSubmitterPullRequestCommentCount', 'InversedDependencyEcosystemExperienceSubmitterPullRequestSubmissionCount', 'InversedDependencyEcosystemExperienceSubmitterPullRequestSuccessRate', 'InversedDependencyEcosystemExperienceSubmitterPullRequestCommentCount', 'IntraProjectSubmitterIssueSubmissionCount', 'IntraProjectSubmitterIssueCommentCount', 'IntraProjectSharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator', 'IntraProjectSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter', 'IntraProjectSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter', 'EcosystemSharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator', 'EcosystemSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter', 'EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter', 'EcosystemExperienceSubmitterIssueSubmissionCount', 'EcosystemExperienceSubmitterIssueCommentCount', 'DependencyEcosystemExperienceSubmitterIssueSubmissionCount', 'DependencyEcosystemExperienceSubmitterIssueCommentCount', 'NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount', 'NonDependencyEcosystemExperienceSubmitterIssueCommentCount', 'InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount', 'InversedDependencyEcosystemExperienceSubmitterIssueCommentCount']\n"
     ]
    }
   ],
   "source": [
    "print(list(df.columns))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct first-time contributor\n",
    "\n",
    "The FTC field calculated by `sliding_window_3` is by definition wrong (because it's multithreaded and FTC is calculated using a global operator whereas each thread only sees a subset of the data). This error is corrected here.\n",
    "\n",
    "It can be done in two fashions: 1, with a pre-existing dataset (e.g., one generated by `sliding_window_2`), or 2, by inferring it from the data. It picks a method automatically based on whether a pre-existing file exists. Because inferring it is a little slow, an FTC data file is created, which can be re-used in any future runs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:31.879651Z",
     "iopub.status.busy": "2024-02-21T16:38:31.879536Z",
     "iopub.status.idle": "2024-02-21T16:38:31.883634Z",
     "shell.execute_reply": "2024-02-21T16:38:31.883336Z"
    }
   },
   "outputs": [],
   "source": [
    "def replace(\n",
    "    df1: pd.DataFrame, df2: pd.DataFrame, match_column: str, overwritten_column: str\n",
    ") -> pd.DataFrame:\n",
    "    replace_data = df2[[match_column, overwritten_column]]\n",
    "\n",
    "    old_counts = df1[overwritten_column].value_counts()\n",
    "\n",
    "    df1 = df1.merge(\n",
    "        replace_data, how=\"left\", on=match_column, suffixes=(\"_incorrect\", \"\")\n",
    "    )\n",
    "    old_key = f\"{overwritten_column}_incorrect\"\n",
    "    df1 = df1.drop(old_key, axis=1)\n",
    "\n",
    "    new_counts = df1[overwritten_column].value_counts()\n",
    "\n",
    "    diff = new_counts - old_counts\n",
    "    print(f\"Impact overwriting '{overwritten_column}':\\n{diff}\")\n",
    "\n",
    "    return df1\n",
    "\n",
    "\n",
    "pre_existing_ftc_data_path = f\"{exp_utils.BASE_PATH}/final_data/ftc_data.csv\"\n",
    "ftc_key = \"SubmitterIsFirstTimeContributor\"\n",
    "\n",
    "\n",
    "def replace_with_pre_existing_data():\n",
    "    global df\n",
    "    print(f\"{pre_existing_ftc_data_path=}\\n\")\n",
    "\n",
    "    other_df: pd.DataFrame = pd.read_csv(pre_existing_ftc_data_path)\n",
    "    df = replace(df, other_df, \"ID\", ftc_key)\n",
    "\n",
    "    del other_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:31.885056Z",
     "iopub.status.busy": "2024-02-21T16:38:31.884935Z",
     "iopub.status.idle": "2024-02-21T16:38:31.888120Z",
     "shell.execute_reply": "2024-02-21T16:38:31.887842Z"
    }
   },
   "outputs": [],
   "source": [
    "def replace_without_pre_existing_data():\n",
    "    \"\"\"\n",
    "    Simply tracks any contributor who is not an FTC,\n",
    "    and if someone is, it checks with the tracked list.\n",
    "    If they are in the list, this means the variable is,\n",
    "    wrong and it's set to False.\n",
    "    \"\"\"\n",
    "\n",
    "    global df\n",
    "\n",
    "    non_ftcs = set()\n",
    "\n",
    "    total_updated = 0\n",
    "    for index, row in df.iterrows():\n",
    "        # It can only illegally go back to FTC,\n",
    "        # so other cases don't need to be checked.\n",
    "        is_ftc = row[ftc_key]\n",
    "        if not is_ftc:\n",
    "            continue\n",
    "\n",
    "        project_name = row[\"Project Name\"]\n",
    "        submitter_id = row[\"Submitter ID\"]\n",
    "        key = hash(f\"{project_name}:{submitter_id}\")\n",
    "\n",
    "        if key in non_ftcs:\n",
    "            total_updated += 1\n",
    "            df.at[index, ftc_key] = False\n",
    "        else:\n",
    "            non_ftcs.add(key)\n",
    "\n",
    "    print(f\"{total_updated=}\")\n",
    "\n",
    "    ftc_df: pd.DataFrame = df[[*df.columns[:6], ftc_key]]\n",
    "    ftc_df.to_csv(pre_existing_ftc_data_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:31.889596Z",
     "iopub.status.busy": "2024-02-21T16:38:31.889480Z",
     "iopub.status.idle": "2024-02-21T16:38:31.891794Z",
     "shell.execute_reply": "2024-02-21T16:38:31.891497Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using FTC data file.\n",
      "pre_existing_ftc_data_path='/workspaces/msc_thesis/data/final_data/ftc_data.csv'\n",
      "\n",
      "Impact overwriting 'SubmitterIsFirstTimeContributor':\n",
      "SubmitterIsFirstTimeContributor\n",
      "False    215739\n",
      "True    -215739\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "from os import path\n",
    "\n",
    "# This step is skipped altogether as it assumes\n",
    "# the obfuscated dataset is accurate.\n",
    "if not using_obfuscated_data:\n",
    "    if path.exists(pre_existing_ftc_data_path):\n",
    "        print(\"Using FTC data file.\")\n",
    "        replace_with_pre_existing_data()\n",
    "    else:\n",
    "        print(\"Inferring FTC data.\")\n",
    "        replace_without_pre_existing_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct second-order degree centrality\n",
    "\n",
    "Calculates 2nd-order degree centrality by aggregating the 2nd-order in-/out-degree values weighing them according to their prevalence, using normalized reciprocal of the edge count; i.e., such that more common edges are weighted less.\n",
    "\n",
    "_Note, the fields are named 1st-order degree centrality. This is a mistake, but I don't want to rewrite all the data retrieval code._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate edge weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:31.893244Z",
     "iopub.status.busy": "2024-02-21T16:38:31.893130Z",
     "iopub.status.idle": "2024-02-21T16:38:31.896537Z",
     "shell.execute_reply": "2024-02-21T16:38:31.896248Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Edge weights:\n",
      " {\n",
      "    \"PRIntegratorToSubmitterV2\": 0.24757886379195637,\n",
      "    \"PRCommenterToSubmitterV2\": 0.24348316399717174,\n",
      "    \"PRCommenterToCommenterV2\": 0.19293707610471117,\n",
      "    \"IssueCommenterToCommenterV2\": 0.07947333160781547,\n",
      "    \"IssueCommenterToSubmitterV2\": 0.23652756449834528\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "# NOTE: There's nothing automated about these numbers;\n",
    "# I copy-pasted them from the terminal after running ``sliding_window_3.py`` on the dataset.\n",
    "# The order of these scores DOES MATTER (the link intensity features assume order).\n",
    "edge_counts = {\n",
    "    \"PRIntegratorToSubmitterV2\": 1043559,\n",
    "    \"PRCommenterToSubmitterV2\": 2808889,\n",
    "    \"PRCommenterToCommenterV2\": 24595282,\n",
    "    \"IssueCommenterToCommenterV2\": 73500466,\n",
    "    \"IssueCommenterToSubmitterV2\": 5806894,\n",
    "}\n",
    "total_edges = sum(edge_counts.values())\n",
    "\n",
    "\n",
    "def calculate_weight(value):\n",
    "    \"\"\"Calculates normalized complement of the weight.\"\"\"\n",
    "    percentage_complement = 1 - (value / (total_edges))\n",
    "    normalized_perc_compl = percentage_complement / (len(edge_counts) - 1)\n",
    "    return normalized_perc_compl\n",
    "\n",
    "\n",
    "edge_weights = {key: calculate_weight(value) for key, value in edge_counts.items()}\n",
    "total_weight = sum(edge_weights.values())\n",
    "edge_weights = {key: value / total_weight for key, value in edge_weights.items()}\n",
    "\n",
    "print(\"Edge weights:\\n\", json.dumps(edge_weights, indent=4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate centrality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:31.898075Z",
     "iopub.status.busy": "2024-02-21T16:38:31.897842Z",
     "iopub.status.idle": "2024-02-21T16:38:50.710658Z",
     "shell.execute_reply": "2024-02-21T16:38:50.710312Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(df.columns)=59\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WeightedEcosystemSecondOrderInDegreeCentrality</th>\n",
       "      <th>WeightedEcosystemSecondOrderOutDegreeCentrality</th>\n",
       "      <th>WeightedIntraProjectSecondOrderInDegreeCentrality</th>\n",
       "      <th>WeightedIntraProjectSecondOrderOutDegreeCentrality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.258389e+01</td>\n",
       "      <td>1.038033e+01</td>\n",
       "      <td>3.753797e+01</td>\n",
       "      <td>4.061421e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.875369e+02</td>\n",
       "      <td>5.176232e+02</td>\n",
       "      <td>1.609692e+03</td>\n",
       "      <td>2.403549e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.567563e-01</td>\n",
       "      <td>4.697693e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.027667e+00</td>\n",
       "      <td>5.467396e-01</td>\n",
       "      <td>2.462693e+00</td>\n",
       "      <td>1.611341e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.811220e+05</td>\n",
       "      <td>2.723745e+05</td>\n",
       "      <td>1.035309e+06</td>\n",
       "      <td>1.488838e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       WeightedEcosystemSecondOrderInDegreeCentrality   \n",
       "count                                    1.815972e+06  \\\n",
       "mean                                     1.258389e+01   \n",
       "std                                      5.875369e+02   \n",
       "min                                      0.000000e+00   \n",
       "25%                                      0.000000e+00   \n",
       "50%                                      0.000000e+00   \n",
       "75%                                      1.027667e+00   \n",
       "max                                      2.811220e+05   \n",
       "\n",
       "       WeightedEcosystemSecondOrderOutDegreeCentrality   \n",
       "count                                     1.815972e+06  \\\n",
       "mean                                      1.038033e+01   \n",
       "std                                       5.176232e+02   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       0.000000e+00   \n",
       "75%                                       5.467396e-01   \n",
       "max                                       2.723745e+05   \n",
       "\n",
       "       WeightedIntraProjectSecondOrderInDegreeCentrality   \n",
       "count                                       1.815972e+06  \\\n",
       "mean                                        3.753797e+01   \n",
       "std                                         1.609692e+03   \n",
       "min                                         0.000000e+00   \n",
       "25%                                         0.000000e+00   \n",
       "50%                                         1.567563e-01   \n",
       "75%                                         2.462693e+00   \n",
       "max                                         1.035309e+06   \n",
       "\n",
       "       WeightedIntraProjectSecondOrderOutDegreeCentrality  \n",
       "count                                       1.815972e+06   \n",
       "mean                                        4.061421e+01   \n",
       "std                                         2.403549e+03   \n",
       "min                                         0.000000e+00   \n",
       "25%                                         0.000000e+00   \n",
       "50%                                         4.697693e-02   \n",
       "75%                                         1.611341e+00   \n",
       "max                                         1.488838e+06   "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from itertools import product\n",
    "\n",
    "scopes = [\"Ecosystem\", \"IntraProject\"]\n",
    "edge_types = edge_weights.keys()\n",
    "directions = [\"In\", \"Out\"]\n",
    "combinations = product(scopes, edge_types, edge_types, directions)\n",
    "\n",
    "BASE_EDGE_FIELD = \"{scope}SecondOrder{direction}DegreeCentrality({connecting_edge}.{experience_edge}-{direction})\"\n",
    "BASE_TARGET_FIELD = \"Weighted{scope}SecondOrder{direction}DegreeCentrality\"\n",
    "\n",
    "centrality_features = []\n",
    "\n",
    "# Iterates through all edge variables.\n",
    "for scope, connecting_edge, experience_edge, direction in combinations:\n",
    "    edge_field = BASE_EDGE_FIELD.format(\n",
    "        scope=scope,\n",
    "        connecting_edge=connecting_edge,\n",
    "        experience_edge=experience_edge,\n",
    "        direction=direction,\n",
    "    )\n",
    "\n",
    "    # Applies the weights to edge.\n",
    "    connecting_weight = edge_weights[connecting_edge]\n",
    "    experience_weight = edge_weights[experience_edge]\n",
    "    total_weight = connecting_weight * experience_weight\n",
    "\n",
    "    df[edge_field] = df[edge_field].multiply(total_weight)\n",
    "\n",
    "    # Add contribution to centrality.\n",
    "    target_field = BASE_TARGET_FIELD.format(scope=scope, direction=direction)\n",
    "    # Sets initial value if it doesn't exist yet.\n",
    "    if target_field not in df.columns:\n",
    "        df[target_field] = 0\n",
    "        centrality_features.append(target_field)\n",
    "\n",
    "    df[target_field] = df[target_field].add(df[edge_field])\n",
    "\n",
    "    # Drops the edge column, as it's only necessary once.\n",
    "    df = df.drop(edge_field, axis=1)\n",
    "\n",
    "print(f\"{len(df.columns)=}\")\n",
    "df[centrality_features].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge Directions\n",
    "\n",
    "Sums the in-degree and out-degree fields as these correlate strongly. This is equivalent to using an undirected graph to calculate this variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:50.712212Z",
     "iopub.status.busy": "2024-02-21T16:38:50.712086Z",
     "iopub.status.idle": "2024-02-21T16:38:50.720736Z",
     "shell.execute_reply": "2024-02-21T16:38:50.720348Z"
    }
   },
   "outputs": [],
   "source": [
    "df[\"WeightedEcosystemSecondOrderDegreeCentrality\"] = df[\n",
    "    \"WeightedEcosystemSecondOrderInDegreeCentrality\"\n",
    "].add(df[\"WeightedEcosystemSecondOrderOutDegreeCentrality\"])\n",
    "\n",
    "df[\"WeightedIntraProjectSecondOrderDegreeCentrality\"] = df[\n",
    "    \"WeightedIntraProjectSecondOrderInDegreeCentrality\"\n",
    "].add(df[\"WeightedIntraProjectSecondOrderOutDegreeCentrality\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:50.722612Z",
     "iopub.status.busy": "2024-02-21T16:38:50.722493Z",
     "iopub.status.idle": "2024-02-21T16:38:50.817721Z",
     "shell.execute_reply": "2024-02-21T16:38:50.817310Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(df.columns)=57\n"
     ]
    }
   ],
   "source": [
    "degree_fields = [\n",
    "    \"WeightedIntraProjectSecondOrderInDegreeCentrality\",\n",
    "    \"WeightedIntraProjectSecondOrderOutDegreeCentrality\",\n",
    "    \"WeightedEcosystemSecondOrderInDegreeCentrality\",\n",
    "    \"WeightedEcosystemSecondOrderOutDegreeCentrality\",\n",
    "]\n",
    "\n",
    "df = df.drop(degree_fields, axis=1)\n",
    "\n",
    "print(f'{len(df.columns)=}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct Link intensity\n",
    "\n",
    "Calculates link intensity by aggregating the individual fields, weighting them using the normalized reciprocal of the edge count (the same weights as used for 2nd-order degree centrality)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculates link intensity\n",
    "\n",
    "Calculates at an ecosystem level and intra-project.\n",
    "Calculated in two directions: \"integrator-to-submitter\" and \"submitter-to-integrator\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:50.819295Z",
     "iopub.status.busy": "2024-02-21T16:38:50.819176Z",
     "iopub.status.idle": "2024-02-21T16:38:51.263447Z",
     "shell.execute_reply": "2024-02-21T16:38:51.263103Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list(flt)=['EcosystemSharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter', 'EcosystemSharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter', 'EcosystemSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter', 'EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter', 'EcosystemSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter', 'EcosystemSharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator', 'EcosystemSharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator', 'EcosystemSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter', 'EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter', 'EcosystemSharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator', 'IntraProjectSharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter', 'IntraProjectSharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter', 'IntraProjectSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter', 'IntraProjectSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter', 'IntraProjectSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter', 'IntraProjectSharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator', 'IntraProjectSharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator', 'IntraProjectSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter', 'IntraProjectSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter', 'IntraProjectSharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EcosystemSharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter</th>\n",
       "      <th>EcosystemSharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter</th>\n",
       "      <th>EcosystemSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>EcosystemSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter</th>\n",
       "      <th>EcosystemSharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator</th>\n",
       "      <th>EcosystemSharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator</th>\n",
       "      <th>EcosystemSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>EcosystemSharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator</th>\n",
       "      <th>IntraProjectSharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter</th>\n",
       "      <th>IntraProjectSharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter</th>\n",
       "      <th>IntraProjectSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>IntraProjectSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>IntraProjectSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter</th>\n",
       "      <th>IntraProjectSharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator</th>\n",
       "      <th>IntraProjectSharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator</th>\n",
       "      <th>IntraProjectSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>IntraProjectSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter</th>\n",
       "      <th>IntraProjectSharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.464026e-01</td>\n",
       "      <td>1.375335e-01</td>\n",
       "      <td>1.691750e+00</td>\n",
       "      <td>1.475467e+00</td>\n",
       "      <td>8.594130e-02</td>\n",
       "      <td>6.086542e-01</td>\n",
       "      <td>3.232561e-01</td>\n",
       "      <td>1.691750e+00</td>\n",
       "      <td>1.475467e+00</td>\n",
       "      <td>1.074113e-01</td>\n",
       "      <td>3.698532e-01</td>\n",
       "      <td>5.043178e-01</td>\n",
       "      <td>1.034282e+01</td>\n",
       "      <td>6.454902e+00</td>\n",
       "      <td>3.165836e-01</td>\n",
       "      <td>2.240939e+00</td>\n",
       "      <td>1.812802e+00</td>\n",
       "      <td>1.034282e+01</td>\n",
       "      <td>6.454902e+00</td>\n",
       "      <td>4.785481e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.440522e+00</td>\n",
       "      <td>1.690763e+00</td>\n",
       "      <td>2.098948e+01</td>\n",
       "      <td>2.382904e+01</td>\n",
       "      <td>1.285316e+00</td>\n",
       "      <td>8.698811e+00</td>\n",
       "      <td>2.922402e+00</td>\n",
       "      <td>2.098948e+01</td>\n",
       "      <td>2.382904e+01</td>\n",
       "      <td>1.372646e+00</td>\n",
       "      <td>3.130804e+00</td>\n",
       "      <td>6.028035e+00</td>\n",
       "      <td>1.255322e+02</td>\n",
       "      <td>6.933401e+01</td>\n",
       "      <td>2.652809e+00</td>\n",
       "      <td>8.967859e+00</td>\n",
       "      <td>8.827352e+00</td>\n",
       "      <td>1.255322e+02</td>\n",
       "      <td>6.933401e+01</td>\n",
       "      <td>3.087171e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.400000e+02</td>\n",
       "      <td>1.450000e+02</td>\n",
       "      <td>3.675000e+03</td>\n",
       "      <td>5.687000e+03</td>\n",
       "      <td>2.190000e+02</td>\n",
       "      <td>5.540000e+02</td>\n",
       "      <td>2.490000e+02</td>\n",
       "      <td>3.675000e+03</td>\n",
       "      <td>5.687000e+03</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>1.610000e+02</td>\n",
       "      <td>9.400000e+02</td>\n",
       "      <td>1.959600e+04</td>\n",
       "      <td>1.951100e+04</td>\n",
       "      <td>2.110000e+02</td>\n",
       "      <td>2.030000e+02</td>\n",
       "      <td>5.650000e+02</td>\n",
       "      <td>1.959600e+04</td>\n",
       "      <td>1.951100e+04</td>\n",
       "      <td>2.570000e+02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       EcosystemSharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter   \n",
       "count                                       1.815972e+06                               \\\n",
       "mean                                        1.464026e-01                                \n",
       "std                                         3.440522e+00                                \n",
       "min                                         0.000000e+00                                \n",
       "25%                                         0.000000e+00                                \n",
       "50%                                         0.000000e+00                                \n",
       "75%                                         0.000000e+00                                \n",
       "max                                         5.400000e+02                                \n",
       "\n",
       "       EcosystemSharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter   \n",
       "count                                       1.815972e+06                                \\\n",
       "mean                                        1.375335e-01                                 \n",
       "std                                         1.690763e+00                                 \n",
       "min                                         0.000000e+00                                 \n",
       "25%                                         0.000000e+00                                 \n",
       "50%                                         0.000000e+00                                 \n",
       "75%                                         0.000000e+00                                 \n",
       "max                                         1.450000e+02                                 \n",
       "\n",
       "       EcosystemSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                                    \\\n",
       "mean                                        1.691750e+00                                     \n",
       "std                                         2.098948e+01                                     \n",
       "min                                         0.000000e+00                                     \n",
       "25%                                         0.000000e+00                                     \n",
       "50%                                         0.000000e+00                                     \n",
       "75%                                         0.000000e+00                                     \n",
       "max                                         3.675000e+03                                     \n",
       "\n",
       "       EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                              \\\n",
       "mean                                        1.475467e+00                               \n",
       "std                                         2.382904e+01                               \n",
       "min                                         0.000000e+00                               \n",
       "25%                                         0.000000e+00                               \n",
       "50%                                         0.000000e+00                               \n",
       "75%                                         0.000000e+00                               \n",
       "max                                         5.687000e+03                               \n",
       "\n",
       "       EcosystemSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter   \n",
       "count                                       1.815972e+06                          \\\n",
       "mean                                        8.594130e-02                           \n",
       "std                                         1.285316e+00                           \n",
       "min                                         0.000000e+00                           \n",
       "25%                                         0.000000e+00                           \n",
       "50%                                         0.000000e+00                           \n",
       "75%                                         0.000000e+00                           \n",
       "max                                         2.190000e+02                           \n",
       "\n",
       "       EcosystemSharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator   \n",
       "count                                       1.815972e+06                               \\\n",
       "mean                                        6.086542e-01                                \n",
       "std                                         8.698811e+00                                \n",
       "min                                         0.000000e+00                                \n",
       "25%                                         0.000000e+00                                \n",
       "50%                                         0.000000e+00                                \n",
       "75%                                         0.000000e+00                                \n",
       "max                                         5.540000e+02                                \n",
       "\n",
       "       EcosystemSharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator   \n",
       "count                                       1.815972e+06                                \\\n",
       "mean                                        3.232561e-01                                 \n",
       "std                                         2.922402e+00                                 \n",
       "min                                         0.000000e+00                                 \n",
       "25%                                         0.000000e+00                                 \n",
       "50%                                         0.000000e+00                                 \n",
       "75%                                         0.000000e+00                                 \n",
       "max                                         2.490000e+02                                 \n",
       "\n",
       "       EcosystemSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                                    \\\n",
       "mean                                        1.691750e+00                                     \n",
       "std                                         2.098948e+01                                     \n",
       "min                                         0.000000e+00                                     \n",
       "25%                                         0.000000e+00                                     \n",
       "50%                                         0.000000e+00                                     \n",
       "75%                                         0.000000e+00                                     \n",
       "max                                         3.675000e+03                                     \n",
       "\n",
       "       EcosystemSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                              \\\n",
       "mean                                        1.475467e+00                               \n",
       "std                                         2.382904e+01                               \n",
       "min                                         0.000000e+00                               \n",
       "25%                                         0.000000e+00                               \n",
       "50%                                         0.000000e+00                               \n",
       "75%                                         0.000000e+00                               \n",
       "max                                         5.687000e+03                               \n",
       "\n",
       "       EcosystemSharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator   \n",
       "count                                       1.815972e+06                          \\\n",
       "mean                                        1.074113e-01                           \n",
       "std                                         1.372646e+00                           \n",
       "min                                         0.000000e+00                           \n",
       "25%                                         0.000000e+00                           \n",
       "50%                                         0.000000e+00                           \n",
       "75%                                         0.000000e+00                           \n",
       "max                                         2.550000e+02                           \n",
       "\n",
       "       IntraProjectSharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter   \n",
       "count                                       1.815972e+06                                  \\\n",
       "mean                                        3.698532e-01                                   \n",
       "std                                         3.130804e+00                                   \n",
       "min                                         0.000000e+00                                   \n",
       "25%                                         0.000000e+00                                   \n",
       "50%                                         0.000000e+00                                   \n",
       "75%                                         0.000000e+00                                   \n",
       "max                                         1.610000e+02                                   \n",
       "\n",
       "       IntraProjectSharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter   \n",
       "count                                       1.815972e+06                                   \\\n",
       "mean                                        5.043178e-01                                    \n",
       "std                                         6.028035e+00                                    \n",
       "min                                         0.000000e+00                                    \n",
       "25%                                         0.000000e+00                                    \n",
       "50%                                         0.000000e+00                                    \n",
       "75%                                         0.000000e+00                                    \n",
       "max                                         9.400000e+02                                    \n",
       "\n",
       "       IntraProjectSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                                       \\\n",
       "mean                                        1.034282e+01                                        \n",
       "std                                         1.255322e+02                                        \n",
       "min                                         0.000000e+00                                        \n",
       "25%                                         0.000000e+00                                        \n",
       "50%                                         0.000000e+00                                        \n",
       "75%                                         0.000000e+00                                        \n",
       "max                                         1.959600e+04                                        \n",
       "\n",
       "       IntraProjectSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                                 \\\n",
       "mean                                        6.454902e+00                                  \n",
       "std                                         6.933401e+01                                  \n",
       "min                                         0.000000e+00                                  \n",
       "25%                                         0.000000e+00                                  \n",
       "50%                                         0.000000e+00                                  \n",
       "75%                                         0.000000e+00                                  \n",
       "max                                         1.951100e+04                                  \n",
       "\n",
       "       IntraProjectSharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter   \n",
       "count                                       1.815972e+06                             \\\n",
       "mean                                        3.165836e-01                              \n",
       "std                                         2.652809e+00                              \n",
       "min                                         0.000000e+00                              \n",
       "25%                                         0.000000e+00                              \n",
       "50%                                         0.000000e+00                              \n",
       "75%                                         0.000000e+00                              \n",
       "max                                         2.110000e+02                              \n",
       "\n",
       "       IntraProjectSharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator   \n",
       "count                                       1.815972e+06                                  \\\n",
       "mean                                        2.240939e+00                                   \n",
       "std                                         8.967859e+00                                   \n",
       "min                                         0.000000e+00                                   \n",
       "25%                                         0.000000e+00                                   \n",
       "50%                                         0.000000e+00                                   \n",
       "75%                                         0.000000e+00                                   \n",
       "max                                         2.030000e+02                                   \n",
       "\n",
       "       IntraProjectSharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator   \n",
       "count                                       1.815972e+06                                   \\\n",
       "mean                                        1.812802e+00                                    \n",
       "std                                         8.827352e+00                                    \n",
       "min                                         0.000000e+00                                    \n",
       "25%                                         0.000000e+00                                    \n",
       "50%                                         0.000000e+00                                    \n",
       "75%                                         0.000000e+00                                    \n",
       "max                                         5.650000e+02                                    \n",
       "\n",
       "       IntraProjectSharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                                       \\\n",
       "mean                                        1.034282e+01                                        \n",
       "std                                         1.255322e+02                                        \n",
       "min                                         0.000000e+00                                        \n",
       "25%                                         0.000000e+00                                        \n",
       "50%                                         0.000000e+00                                        \n",
       "75%                                         0.000000e+00                                        \n",
       "max                                         1.959600e+04                                        \n",
       "\n",
       "       IntraProjectSharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter   \n",
       "count                                       1.815972e+06                                 \\\n",
       "mean                                        6.454902e+00                                  \n",
       "std                                         6.933401e+01                                  \n",
       "min                                         0.000000e+00                                  \n",
       "25%                                         0.000000e+00                                  \n",
       "50%                                         0.000000e+00                                  \n",
       "75%                                         0.000000e+00                                  \n",
       "max                                         1.951100e+04                                  \n",
       "\n",
       "       IntraProjectSharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator  \n",
       "count                                       1.815972e+06                             \n",
       "mean                                        4.785481e-01                             \n",
       "std                                         3.087171e+00                             \n",
       "min                                         0.000000e+00                             \n",
       "25%                                         0.000000e+00                             \n",
       "50%                                         0.000000e+00                             \n",
       "75%                                         0.000000e+00                             \n",
       "max                                         2.570000e+02                             "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "directions = [\"IntegratorToSubmitter\", \"SubmitterToIntegrator\"]\n",
    "\n",
    "\n",
    "# BUG: Both directions contains the discussion participation fields.\n",
    "# These are thus bound to correlate; this  should be fixed.\n",
    "direction_fields = {\n",
    "    directions[0]: [\n",
    "        \"SharedExperiencePullRequestSubmittedByIntegratorIntegratedBySubmitter\",\n",
    "        \"SharedExperiencePullRequestSubmittedByIntegratorCommentedOnBySubmitter\",\n",
    "        \"SharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter\",\n",
    "        \"SharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter\",\n",
    "        \"SharedExperienceIssueSubmittedByIntegratorCommentedOnBySubmitter\",\n",
    "    ],\n",
    "    directions[1]: [\n",
    "        \"SharedExperiencePullRequestSubmittedBySubmitterIntegratedByIntegrator\",\n",
    "        \"SharedExperiencePullRequestSubmittedBySubmitterCommentedOnByIntegrator\",\n",
    "        \"SharedExperiencePullRequestDiscussionParticipationByIntegratorAndSubmitter\",\n",
    "        \"SharedExperienceIssueDiscussionParticipationByIntegratorAndSubmitter\",\n",
    "        \"SharedExperienceIssueSubmittedBySubmitterCommentedOnByIntegrator\",\n",
    "    ],\n",
    "}\n",
    "\n",
    "from wmutils.collections.list_access import flatten\n",
    "\n",
    "\n",
    "flt = list(flatten(direction_fields.values()))\n",
    "flt = [f'{scope}{field}' for scope, field in product(scopes, flt) if f'{scope}{field}' in df.columns]\n",
    "print(f'{list(flt)=}')\n",
    "\n",
    "df[flt].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:51.265266Z",
     "iopub.status.busy": "2024-02-21T16:38:51.265147Z",
     "iopub.status.idle": "2024-02-21T16:38:51.586091Z",
     "shell.execute_reply": "2024-02-21T16:38:51.585727Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(df.columns)=45\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EcosystemIntegratorToSubmitterLinkIntensity</th>\n",
       "      <th>EcosystemSubmitterToIntegratorLinkIntensity</th>\n",
       "      <th>IntraProjectIntegratorToSubmitterLinkIntensity</th>\n",
       "      <th>IntraProjectSubmitterToIntegratorLinkIntensity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>5.337223e-01</td>\n",
       "      <td>3.270971e-01</td>\n",
       "      <td>2.797748e+00</td>\n",
       "      <td>1.535164e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.752642e+00</td>\n",
       "      <td>2.982510e+00</td>\n",
       "      <td>2.890049e+01</td>\n",
       "      <td>7.638521e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.475789e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.090438e+02</td>\n",
       "      <td>1.957852e+02</td>\n",
       "      <td>4.432496e+03</td>\n",
       "      <td>8.173781e+02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       EcosystemIntegratorToSubmitterLinkIntensity   \n",
       "count                                 1.815972e+06  \\\n",
       "mean                                  5.337223e-01   \n",
       "std                                   5.752642e+00   \n",
       "min                                   0.000000e+00   \n",
       "25%                                   0.000000e+00   \n",
       "50%                                   0.000000e+00   \n",
       "75%                                   0.000000e+00   \n",
       "max                                   7.090438e+02   \n",
       "\n",
       "       EcosystemSubmitterToIntegratorLinkIntensity   \n",
       "count                                 1.815972e+06  \\\n",
       "mean                                  3.270971e-01   \n",
       "std                                   2.982510e+00   \n",
       "min                                   0.000000e+00   \n",
       "25%                                   0.000000e+00   \n",
       "50%                                   0.000000e+00   \n",
       "75%                                   0.000000e+00   \n",
       "max                                   1.957852e+02   \n",
       "\n",
       "       IntraProjectIntegratorToSubmitterLinkIntensity   \n",
       "count                                    1.815972e+06  \\\n",
       "mean                                     2.797748e+00   \n",
       "std                                      2.890049e+01   \n",
       "min                                      0.000000e+00   \n",
       "25%                                      0.000000e+00   \n",
       "50%                                      0.000000e+00   \n",
       "75%                                      0.000000e+00   \n",
       "max                                      4.432496e+03   \n",
       "\n",
       "       IntraProjectSubmitterToIntegratorLinkIntensity  \n",
       "count                                    1.815972e+06  \n",
       "mean                                     1.535164e+00  \n",
       "std                                      7.638521e+00  \n",
       "min                                      0.000000e+00  \n",
       "25%                                      0.000000e+00  \n",
       "50%                                      0.000000e+00  \n",
       "75%                                      2.475789e-01  \n",
       "max                                      8.173781e+02  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BASE_TARGET_FIELD = \"{scope}{direction}LinkIntensity\"\n",
    "\n",
    "combinations = product(scopes, directions)\n",
    "link_intensity_fields = []\n",
    "\n",
    "directional_link_intensity_fields = set()\n",
    "\n",
    "edge_weights_list = list(edge_weights.values())\n",
    "for scope, direction in combinations:\n",
    "    target_field = BASE_TARGET_FIELD.format(scope=scope, direction=direction)\n",
    "    fields = direction_fields[direction]\n",
    "\n",
    "    # Adds initial value for the field.\n",
    "    df[target_field] = 0\n",
    "    link_intensity_fields.append(target_field)\n",
    "\n",
    "    for index, field in enumerate(fields):\n",
    "        field = f\"{scope}{field}\"\n",
    "\n",
    "        # Applies weight\n",
    "        field_weight = edge_weights_list[index]\n",
    "        df[field] = df[field].multiply(field_weight)\n",
    "\n",
    "        # Adds field contribution to link intensity.\n",
    "        df[target_field] = df[target_field].add(df[field])\n",
    "\n",
    "        # Drops field as its only used once.\n",
    "        directional_link_intensity_fields.add(field)\n",
    "        # df = df.drop(field, axis=1)\n",
    "\n",
    "df = df.drop(directional_link_intensity_fields, axis=1)\n",
    "print(f\"{len(df.columns)=}\")\n",
    "df[link_intensity_fields].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merging Directions\n",
    "\n",
    "Sums the \"integrator to submitter\" and \"submitter to integrator\" fields as these correlate strongly. This is equivalent to using an undirected graph to calculate this variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:51.587652Z",
     "iopub.status.busy": "2024-02-21T16:38:51.587503Z",
     "iopub.status.idle": "2024-02-21T16:38:51.596247Z",
     "shell.execute_reply": "2024-02-21T16:38:51.595804Z"
    }
   },
   "outputs": [],
   "source": [
    "df[\"EcosystemLinkIntensity\"] = df[\"EcosystemIntegratorToSubmitterLinkIntensity\"].add(\n",
    "    df[\"EcosystemSubmitterToIntegratorLinkIntensity\"]\n",
    ")\n",
    "\n",
    "df[\"IntraProjectLinkIntensity\"] = df[\n",
    "    \"IntraProjectIntegratorToSubmitterLinkIntensity\"\n",
    "].add(df[\"IntraProjectSubmitterToIntegratorLinkIntensity\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:51.598300Z",
     "iopub.status.busy": "2024-02-21T16:38:51.598120Z",
     "iopub.status.idle": "2024-02-21T16:38:51.669795Z",
     "shell.execute_reply": "2024-02-21T16:38:51.669310Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(df.columns)=43\n"
     ]
    }
   ],
   "source": [
    "directional_link_intensity_fields = [\n",
    "    \"EcosystemIntegratorToSubmitterLinkIntensity\",\n",
    "    \"EcosystemSubmitterToIntegratorLinkIntensity\",\n",
    "    \"IntraProjectIntegratorToSubmitterLinkIntensity\",\n",
    "    \"IntraProjectSubmitterToIntegratorLinkIntensity\",\n",
    "]\n",
    "\n",
    "df = df.drop(directional_link_intensity_fields, axis=1)\n",
    "\n",
    "print(f\"{len(df.columns)=}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save dataset to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:38:51.671410Z",
     "iopub.status.busy": "2024-02-21T16:38:51.671287Z",
     "iopub.status.idle": "2024-02-21T16:39:08.017357Z",
     "shell.execute_reply": "2024-02-21T16:39:08.017007Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Outputting to '/workspaces/msc_thesis/data/final_data/dataset_with_constructed.csv'\n",
      "1815972\n",
      "43\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Submitter ID</th>\n",
       "      <th>PR Number</th>\n",
       "      <th>ControlPullRequestLifeTimeInMinutes</th>\n",
       "      <th>ControlNumberOfCommitsInPullRequest</th>\n",
       "      <th>ControlIntraProjectPullRequestExperienceOfIntegrator</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSubmissionCount</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSuccessRate</th>\n",
       "      <th>IntraProjectSubmitterPullRequestCommentCount</th>\n",
       "      <th>EcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>...</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>WeightedEcosystemSecondOrderDegreeCentrality</th>\n",
       "      <th>WeightedIntraProjectSecondOrderDegreeCentrality</th>\n",
       "      <th>EcosystemLinkIntensity</th>\n",
       "      <th>IntraProjectLinkIntensity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "      <td>1.815972e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.474751e+08</td>\n",
       "      <td>5.198276e+06</td>\n",
       "      <td>3.807207e+03</td>\n",
       "      <td>3.190111e+04</td>\n",
       "      <td>4.947501e+00</td>\n",
       "      <td>1.932087e+02</td>\n",
       "      <td>1.679350e+01</td>\n",
       "      <td>5.352468e-01</td>\n",
       "      <td>2.170048e+01</td>\n",
       "      <td>3.702776e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>1.743463e-01</td>\n",
       "      <td>8.645552e-01</td>\n",
       "      <td>2.274088e+00</td>\n",
       "      <td>1.570265e+01</td>\n",
       "      <td>1.854588e-01</td>\n",
       "      <td>1.355166e+00</td>\n",
       "      <td>2.296422e+01</td>\n",
       "      <td>7.815218e+01</td>\n",
       "      <td>8.608194e-01</td>\n",
       "      <td>4.332911e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.004294e+08</td>\n",
       "      <td>8.100370e+06</td>\n",
       "      <td>8.071615e+03</td>\n",
       "      <td>1.399335e+05</td>\n",
       "      <td>7.283362e+01</td>\n",
       "      <td>4.827151e+02</td>\n",
       "      <td>5.548605e+01</td>\n",
       "      <td>4.623260e-01</td>\n",
       "      <td>6.563799e+01</td>\n",
       "      <td>4.472133e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>1.580686e+00</td>\n",
       "      <td>1.186131e+01</td>\n",
       "      <td>7.064174e+00</td>\n",
       "      <td>6.131080e+01</td>\n",
       "      <td>2.446396e+00</td>\n",
       "      <td>1.667164e+01</td>\n",
       "      <td>1.037335e+03</td>\n",
       "      <td>3.696540e+03</td>\n",
       "      <td>8.019720e+00</td>\n",
       "      <td>3.575078e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>6.870000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.666667e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.807322e+07</td>\n",
       "      <td>4.631570e+05</td>\n",
       "      <td>1.000000e+02</td>\n",
       "      <td>5.290000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.371940e+08</td>\n",
       "      <td>1.620265e+06</td>\n",
       "      <td>5.480000e+02</td>\n",
       "      <td>9.799167e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.000000e+01</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>8.000000e-01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.503077e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.278026e+08</td>\n",
       "      <td>6.384100e+06</td>\n",
       "      <td>2.817000e+03</td>\n",
       "      <td>7.328350e+03</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.410000e+02</td>\n",
       "      <td>1.700000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.500000e+01</td>\n",
       "      <td>9.186047e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.870552e+00</td>\n",
       "      <td>4.883533e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.910620e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.617756e+08</td>\n",
       "      <td>5.976123e+07</td>\n",
       "      <td>8.297600e+04</td>\n",
       "      <td>4.489490e+06</td>\n",
       "      <td>1.000000e+04</td>\n",
       "      <td>4.281000e+03</td>\n",
       "      <td>1.703000e+03</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.038000e+03</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>3.730000e+02</td>\n",
       "      <td>9.270000e+02</td>\n",
       "      <td>2.332000e+03</td>\n",
       "      <td>7.630000e+03</td>\n",
       "      <td>7.410000e+02</td>\n",
       "      <td>2.764000e+03</td>\n",
       "      <td>5.534965e+05</td>\n",
       "      <td>2.524147e+06</td>\n",
       "      <td>9.048289e+02</td>\n",
       "      <td>5.249874e+03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  Submitter ID     PR Number   \n",
       "count  1.815972e+06  1.815972e+06  1.815972e+06  \\\n",
       "mean   1.474751e+08  5.198276e+06  3.807207e+03   \n",
       "std    1.004294e+08  8.100370e+06  8.071615e+03   \n",
       "min    6.870000e+02  1.000000e+00  1.000000e+00   \n",
       "25%    5.807322e+07  4.631570e+05  1.000000e+02   \n",
       "50%    1.371940e+08  1.620265e+06  5.480000e+02   \n",
       "75%    2.278026e+08  6.384100e+06  2.817000e+03   \n",
       "max    3.617756e+08  5.976123e+07  8.297600e+04   \n",
       "\n",
       "       ControlPullRequestLifeTimeInMinutes   \n",
       "count                         1.815972e+06  \\\n",
       "mean                          3.190111e+04   \n",
       "std                           1.399335e+05   \n",
       "min                           1.666667e-02   \n",
       "25%                           5.290000e+01   \n",
       "50%                           9.799167e+02   \n",
       "75%                           7.328350e+03   \n",
       "max                           4.489490e+06   \n",
       "\n",
       "       ControlNumberOfCommitsInPullRequest   \n",
       "count                         1.815972e+06  \\\n",
       "mean                          4.947501e+00   \n",
       "std                           7.283362e+01   \n",
       "min                           0.000000e+00   \n",
       "25%                           1.000000e+00   \n",
       "50%                           1.000000e+00   \n",
       "75%                           3.000000e+00   \n",
       "max                           1.000000e+04   \n",
       "\n",
       "       ControlIntraProjectPullRequestExperienceOfIntegrator   \n",
       "count                                       1.815972e+06     \\\n",
       "mean                                        1.932087e+02      \n",
       "std                                         4.827151e+02      \n",
       "min                                         0.000000e+00      \n",
       "25%                                         5.000000e+00      \n",
       "50%                                         3.000000e+01      \n",
       "75%                                         1.410000e+02      \n",
       "max                                         4.281000e+03      \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSubmissionCount   \n",
       "count                                     1.815972e+06  \\\n",
       "mean                                      1.679350e+01   \n",
       "std                                       5.548605e+01   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       2.000000e+00   \n",
       "75%                                       1.700000e+01   \n",
       "max                                       1.703000e+03   \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSuccessRate   \n",
       "count                                 1.815972e+06  \\\n",
       "mean                                  5.352468e-01   \n",
       "std                                   4.623260e-01   \n",
       "min                                   0.000000e+00   \n",
       "25%                                   0.000000e+00   \n",
       "50%                                   8.000000e-01   \n",
       "75%                                   1.000000e+00   \n",
       "max                                   1.000000e+00   \n",
       "\n",
       "       IntraProjectSubmitterPullRequestCommentCount   \n",
       "count                                  1.815972e+06  \\\n",
       "mean                                   2.170048e+01   \n",
       "std                                    6.563799e+01   \n",
       "min                                    0.000000e+00   \n",
       "25%                                    0.000000e+00   \n",
       "50%                                    1.000000e+00   \n",
       "75%                                    1.500000e+01   \n",
       "max                                    2.038000e+03   \n",
       "\n",
       "       EcosystemExperienceSubmitterPullRequestSuccessRate  ...   \n",
       "count                                       1.815972e+06   ...  \\\n",
       "mean                                        3.702776e-01   ...   \n",
       "std                                         4.472133e-01   ...   \n",
       "min                                         0.000000e+00   ...   \n",
       "25%                                         0.000000e+00   ...   \n",
       "50%                                         0.000000e+00   ...   \n",
       "75%                                         9.186047e-01   ...   \n",
       "max                                         1.000000e+00   ...   \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.815972e+06           \\\n",
       "mean                                        1.743463e-01            \n",
       "std                                         1.580686e+00            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         0.000000e+00            \n",
       "75%                                         0.000000e+00            \n",
       "max                                         3.730000e+02            \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.815972e+06        \\\n",
       "mean                                        8.645552e-01         \n",
       "std                                         1.186131e+01         \n",
       "min                                         0.000000e+00         \n",
       "25%                                         0.000000e+00         \n",
       "50%                                         0.000000e+00         \n",
       "75%                                         0.000000e+00         \n",
       "max                                         9.270000e+02         \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.815972e+06              \\\n",
       "mean                                        2.274088e+00               \n",
       "std                                         7.064174e+00               \n",
       "min                                         0.000000e+00               \n",
       "25%                                         0.000000e+00               \n",
       "50%                                         0.000000e+00               \n",
       "75%                                         2.000000e+00               \n",
       "max                                         2.332000e+03               \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.815972e+06           \\\n",
       "mean                                        1.570265e+01            \n",
       "std                                         6.131080e+01            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         0.000000e+00            \n",
       "75%                                         6.000000e+00            \n",
       "max                                         7.630000e+03            \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.815972e+06                   \\\n",
       "mean                                        1.854588e-01                    \n",
       "std                                         2.446396e+00                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         0.000000e+00                    \n",
       "75%                                         0.000000e+00                    \n",
       "max                                         7.410000e+02                    \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.815972e+06                \\\n",
       "mean                                        1.355166e+00                 \n",
       "std                                         1.667164e+01                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         0.000000e+00                 \n",
       "max                                         2.764000e+03                 \n",
       "\n",
       "       WeightedEcosystemSecondOrderDegreeCentrality   \n",
       "count                                  1.815972e+06  \\\n",
       "mean                                   2.296422e+01   \n",
       "std                                    1.037335e+03   \n",
       "min                                    0.000000e+00   \n",
       "25%                                    0.000000e+00   \n",
       "50%                                    0.000000e+00   \n",
       "75%                                    1.870552e+00   \n",
       "max                                    5.534965e+05   \n",
       "\n",
       "       WeightedIntraProjectSecondOrderDegreeCentrality   \n",
       "count                                     1.815972e+06  \\\n",
       "mean                                      7.815218e+01   \n",
       "std                                       3.696540e+03   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       3.503077e-01   \n",
       "75%                                       4.883533e+00   \n",
       "max                                       2.524147e+06   \n",
       "\n",
       "       EcosystemLinkIntensity  IntraProjectLinkIntensity  \n",
       "count            1.815972e+06               1.815972e+06  \n",
       "mean             8.608194e-01               4.332911e+00  \n",
       "std              8.019720e+00               3.575078e+01  \n",
       "min              0.000000e+00               0.000000e+00  \n",
       "25%              0.000000e+00               0.000000e+00  \n",
       "50%              0.000000e+00               0.000000e+00  \n",
       "75%              0.000000e+00               4.910620e-01  \n",
       "max              9.048289e+02               5.249874e+03  \n",
       "\n",
       "[8 rows x 35 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_data_file_name = \"dataset_with_constructed\"\n",
    "output_path = data_path_format.format(data_file_name=output_data_file_name)\n",
    "print(f\"Outputting to '{output_path}'\")\n",
    "\n",
    "df.to_csv(output_path, index=False)\n",
    "\n",
    "print(f'{len(df)}')\n",
    "print(f'{len(df.columns)}')\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:39:08.019082Z",
     "iopub.status.busy": "2024-02-21T16:39:08.018955Z",
     "iopub.status.idle": "2024-02-21T16:39:08.021271Z",
     "shell.execute_reply": "2024-02-21T16:39:08.020995Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-01-07 09:14:50.426441\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "\n",
    "print(datetime.datetime.now())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}