model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model description\n",
    "**Outcome variable**\n",
    "- price (eth)  \n",
    "\n",
    "**Explanatory variables**\n",
    "- collection features (daily): volume, total trades, price max, price min, price 5%  \n",
    "- last sale: price_last_sale, time_last_sale  \n",
    "- traits/rarity: (1 categorical variable for each trait), rarity rank  \n",
    "\n",
    "**Models used (sklearn)**\n",
    "- Linear Models: OLS, Lasso, Ridge\n",
    "- Ensemble: random forest, gradient boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = \"bayc_cleaned.csv\"\n",
    "df = pd.read_csv(f\"data/{file}\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>project</th>\n",
       "      <th>version</th>\n",
       "      <th>block_date</th>\n",
       "      <th>block_month</th>\n",
       "      <th>block_time</th>\n",
       "      <th>token_id</th>\n",
       "      <th>collection</th>\n",
       "      <th>amount_usd</th>\n",
       "      <th>token_standard</th>\n",
       "      <th>trade_type</th>\n",
       "      <th>...</th>\n",
       "      <th>Mouth_value</th>\n",
       "      <th>Mouth_count</th>\n",
       "      <th>Eyes_value</th>\n",
       "      <th>Eyes_count</th>\n",
       "      <th>Fur_value</th>\n",
       "      <th>Fur_count</th>\n",
       "      <th>Clothes_value</th>\n",
       "      <th>Clothes_count</th>\n",
       "      <th>Earring_value</th>\n",
       "      <th>Earring_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>x2y2</td>\n",
       "      <td>v1</td>\n",
       "      <td>2023-01-22</td>\n",
       "      <td>2023-01-01</td>\n",
       "      <td>2023-01-22 00:30:23+00:00</td>\n",
       "      <td>5</td>\n",
       "      <td>Bored Ape Yacht Club</td>\n",
       "      <td>317355.814</td>\n",
       "      <td>erc721</td>\n",
       "      <td>secondary</td>\n",
       "      <td>...</td>\n",
       "      <td>Dumbfounded</td>\n",
       "      <td>505</td>\n",
       "      <td>X Eyes</td>\n",
       "      <td>243</td>\n",
       "      <td>Brown</td>\n",
       "      <td>1370</td>\n",
       "      <td>Bayc T Red</td>\n",
       "      <td>140</td>\n",
       "      <td>Empty</td>\n",
       "      <td>6383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>opensea</td>\n",
       "      <td>v1</td>\n",
       "      <td>2021-06-21</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>2021-06-21 15:56:28+00:00</td>\n",
       "      <td>6</td>\n",
       "      <td>Bored Ape Yacht Club</td>\n",
       "      <td>19761.700</td>\n",
       "      <td>erc721</td>\n",
       "      <td>Single Item Trade: Fixed price</td>\n",
       "      <td>...</td>\n",
       "      <td>Bored</td>\n",
       "      <td>2272</td>\n",
       "      <td>Crazy</td>\n",
       "      <td>407</td>\n",
       "      <td>Cream</td>\n",
       "      <td>636</td>\n",
       "      <td>Tweed Suit</td>\n",
       "      <td>141</td>\n",
       "      <td>Empty</td>\n",
       "      <td>6383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>opensea</td>\n",
       "      <td>v1</td>\n",
       "      <td>2021-12-22</td>\n",
       "      <td>2021-12-01</td>\n",
       "      <td>2021-12-22 19:16:09+00:00</td>\n",
       "      <td>6</td>\n",
       "      <td>Bored Ape Yacht Club</td>\n",
       "      <td>251182.890</td>\n",
       "      <td>erc721</td>\n",
       "      <td>Single Item Trade: Fixed price</td>\n",
       "      <td>...</td>\n",
       "      <td>Bored</td>\n",
       "      <td>2272</td>\n",
       "      <td>Crazy</td>\n",
       "      <td>407</td>\n",
       "      <td>Cream</td>\n",
       "      <td>636</td>\n",
       "      <td>Tweed Suit</td>\n",
       "      <td>141</td>\n",
       "      <td>Empty</td>\n",
       "      <td>6383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>opensea</td>\n",
       "      <td>v3</td>\n",
       "      <td>2022-07-03</td>\n",
       "      <td>2022-07-01</td>\n",
       "      <td>2022-07-03 16:40:36+00:00</td>\n",
       "      <td>6</td>\n",
       "      <td>Bored Ape Yacht Club</td>\n",
       "      <td>93910.130</td>\n",
       "      <td>erc721</td>\n",
       "      <td>single item trade</td>\n",
       "      <td>...</td>\n",
       "      <td>Bored</td>\n",
       "      <td>2272</td>\n",
       "      <td>Crazy</td>\n",
       "      <td>407</td>\n",
       "      <td>Cream</td>\n",
       "      <td>636</td>\n",
       "      <td>Tweed Suit</td>\n",
       "      <td>141</td>\n",
       "      <td>Empty</td>\n",
       "      <td>6383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>opensea</td>\n",
       "      <td>v3</td>\n",
       "      <td>2022-10-21</td>\n",
       "      <td>2022-10-01</td>\n",
       "      <td>2022-10-21 00:56:23+00:00</td>\n",
       "      <td>8</td>\n",
       "      <td>Bored Ape Yacht Club</td>\n",
       "      <td>141705.300</td>\n",
       "      <td>erc721</td>\n",
       "      <td>single item trade</td>\n",
       "      <td>...</td>\n",
       "      <td>Bored</td>\n",
       "      <td>2272</td>\n",
       "      <td>Robot</td>\n",
       "      <td>350</td>\n",
       "      <td>Zombie</td>\n",
       "      <td>302</td>\n",
       "      <td>Empty</td>\n",
       "      <td>1690</td>\n",
       "      <td>Gold Stud</td>\n",
       "      <td>439</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   project version  block_date block_month                 block_time  \\\n",
       "0     x2y2      v1  2023-01-22  2023-01-01  2023-01-22 00:30:23+00:00   \n",
       "1  opensea      v1  2021-06-21  2021-06-01  2021-06-21 15:56:28+00:00   \n",
       "2  opensea      v1  2021-12-22  2021-12-01  2021-12-22 19:16:09+00:00   \n",
       "3  opensea      v3  2022-07-03  2022-07-01  2022-07-03 16:40:36+00:00   \n",
       "4  opensea      v3  2022-10-21  2022-10-01  2022-10-21 00:56:23+00:00   \n",
       "\n",
       "   token_id            collection  amount_usd token_standard  \\\n",
       "0         5  Bored Ape Yacht Club  317355.814         erc721   \n",
       "1         6  Bored Ape Yacht Club   19761.700         erc721   \n",
       "2         6  Bored Ape Yacht Club  251182.890         erc721   \n",
       "3         6  Bored Ape Yacht Club   93910.130         erc721   \n",
       "4         8  Bored Ape Yacht Club  141705.300         erc721   \n",
       "\n",
       "                       trade_type  ...  Mouth_value Mouth_count Eyes_value  \\\n",
       "0                       secondary  ...  Dumbfounded         505     X Eyes   \n",
       "1  Single Item Trade: Fixed price  ...        Bored        2272      Crazy   \n",
       "2  Single Item Trade: Fixed price  ...        Bored        2272      Crazy   \n",
       "3               single item trade  ...        Bored        2272      Crazy   \n",
       "4               single item trade  ...        Bored        2272      Robot   \n",
       "\n",
       "   Eyes_count Fur_value Fur_count  Clothes_value  Clothes_count  \\\n",
       "0         243     Brown      1370     Bayc T Red            140   \n",
       "1         407     Cream       636     Tweed Suit            141   \n",
       "2         407     Cream       636     Tweed Suit            141   \n",
       "3         407     Cream       636     Tweed Suit            141   \n",
       "4         350    Zombie       302          Empty           1690   \n",
       "\n",
       "   Earring_value  Earring_count  \n",
       "0          Empty           6383  \n",
       "1          Empty           6383  \n",
       "2          Empty           6383  \n",
       "3          Empty           6383  \n",
       "4      Gold Stud            439  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define column sets\n",
    "feature_columns = {\n",
    "    \"market\" : ['volume_eth', 'price_p5_eth_lag1', 'price_max_eth_lag1', 'price_min_eth_lag1'],\n",
    "    \"traits_rarity\" : ['rarity_rank', 'Background_count', 'Mouth_count', 'Eyes_count', 'Fur_count', 'Hat_count', 'Clothes_count', 'Earring_count'],\n",
    "    \"last_trade\" : ['last_trade_timediff', 'last_trade_price'],\n",
    "}\n",
    "\n",
    "outcome_columns = {\n",
    "    'trade_price' : 'trade_price',\n",
    "    'amount_usd' : 'amount_usd',\n",
    "    'trade_price_log' : 'trade_price_log'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_to_xy(df, X_cols, y_col):\n",
    "    \n",
    "    # fetch variables of interest, drop missing values\n",
    "    data = df.copy()\n",
    "    n = len(data)\n",
    "    data = data.dropna(subset=X_cols+[y_col])\n",
    "    print(f\"Number of observations: {len(data)}/{n}\")\n",
    "    \n",
    "    X = data[X_cols]\n",
    "    y = data[y_col]\n",
    "    \n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark models\n",
    "- Linear regression: OLS, Ridge, Lasso\n",
    "- Tree-based model: Random Forest, gradient boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression, Lasso, Ridge\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "\n",
    "def benchmark_models(df, X_cols, y_col, models=None):\n",
    "    \n",
    "    if models is None:\n",
    "        models = [\n",
    "            'ols', \n",
    "            'lasso',\n",
    "            'ridge',\n",
    "            'rf',\n",
    "            'gb'\n",
    "            ]\n",
    "\n",
    "    X,y = df_to_xy(df,X_cols,y_col)\n",
    "\n",
    "    # train-test split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "    \n",
    "    # standardize\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "\n",
    "    # OLS regression\n",
    "    if 'ols' in models:\n",
    "        ols = LinearRegression()\n",
    "        ols.fit(X_train,y_train)\n",
    "        ols_pred = ols.predict(X_test)\n",
    "        ols_mse = mean_squared_error(y_test, ols_pred)\n",
    "        print(f\"OLS MSE: {ols_mse}\")\n",
    "\n",
    "    # Lasso regression\n",
    "    if \"lasso\" in models:\n",
    "        lasso = Lasso(random_state=42)\n",
    "        lasso.fit(X_train,y_train)\n",
    "        lasso_pred = lasso.predict(X_test)\n",
    "        lasso_mse = mean_squared_error(y_test, lasso_pred)\n",
    "        print(f\"Lasso MSE: {lasso_mse}\")\n",
    "    \n",
    "    # Ridge regression\n",
    "    if \"ridge\" in models:\n",
    "        ridge = Ridge()\n",
    "        ridge.fit(X_train,y_train)\n",
    "        ridge_pred = ridge.predict(X_test)\n",
    "        ridge_mse = mean_squared_error(y_test, ridge_pred)\n",
    "        print(f\"Ridge MSE: {ridge_mse}\")\n",
    "    \n",
    "    # Random Forest regression\n",
    "    if 'rf' in models:\n",
    "        rf = RandomForestRegressor(random_state=42)\n",
    "        rf.fit(X_train, y_train)\n",
    "        rf_predictions = rf.predict(X_test)\n",
    "        rf_mse = mean_squared_error(y_test, rf_predictions)\n",
    "        print(f\"Random Forest MSE: {rf_mse}\")\n",
    "    \n",
    "    # Gradient Boosting regression\n",
    "    if \"gb\" in models:\n",
    "        gb = GradientBoostingRegressor(random_state=42)\n",
    "        gb.fit(X_train, y_train)\n",
    "        gb_predictions = gb.predict(X_test)\n",
    "        gb_mse = mean_squared_error(y_test, gb_predictions)\n",
    "        print(f\"Gradient Boosting MSE: {gb_mse}\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'feature_columns' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)\n",
      "Cell \u001b[1;32mIn[4], line 2\u001b[0m\n",
      "\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# model 1\u001b[39;00m\n",
      "\u001b[1;32m----> 2\u001b[0m X_cols_1 \u001b[38;5;241m=\u001b[39m \u001b[43mfeature_columns\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmarket\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m+\u001b[39m feature_columns[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtraits_rarity\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m+\u001b[39m feature_columns[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlast_trade\u001b[39m\u001b[38;5;124m'\u001b[39m]\n",
      "\u001b[0;32m      3\u001b[0m X_cols_2 \u001b[38;5;241m=\u001b[39m feature_columns[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmarket\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m+\u001b[39m feature_columns[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtraits_rarity\u001b[39m\u001b[38;5;124m'\u001b[39m]\n",
      "\u001b[0;32m      4\u001b[0m X_cols_3 \u001b[38;5;241m=\u001b[39m feature_columns[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtraits_rarity\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m+\u001b[39m feature_columns[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlast_trade\u001b[39m\u001b[38;5;124m'\u001b[39m]\n",
      "\n",
      "\u001b[1;31mNameError\u001b[0m: name 'feature_columns' is not defined"
     ]
    }
   ],
   "source": [
    "# model 1\n",
    "X_cols_1 = feature_columns['market'] + feature_columns['traits_rarity'] + feature_columns['last_trade']\n",
    "X_cols_2 = feature_columns['market'] + feature_columns['traits_rarity']\n",
    "X_cols_3 = feature_columns['traits_rarity'] + feature_columns['last_trade']\n",
    "\n",
    "y_col_1 = \"trade_price\"\n",
    "y_col_2 = \"trade_price_log\"\n",
    "y_col_3 = \"amount_usd\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example\n",
    "predict trade price on base cols, base traits, and last trade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 1\n",
      "Outcome variable: trade_price\n",
      "Explanatory variable set: X_1\n",
      "Number of observations: 34344/43383\n",
      "OLS MSE: 353.017005002352\n",
      "Lasso MSE: 374.1097891057342\n",
      "Ridge MSE: 353.0173740891603\n",
      "Random Forest MSE: 154.96828490789568\n",
      "Gradient Boosting MSE: 143.20387560903973\n",
      "\n",
      "\n",
      "Model 2\n",
      "Outcome variable: trade_price\n",
      "Explanatory variable set: X_2\n",
      "Number of observations: 43383/43383\n",
      "OLS MSE: 319.0132516862824\n",
      "Lasso MSE: 327.36327523108525\n",
      "Ridge MSE: 319.0129306209468\n",
      "Random Forest MSE: 200.96036864079136\n",
      "Gradient Boosting MSE: 200.50187412353847\n",
      "\n",
      "\n",
      "Model 3\n",
      "Outcome variable: trade_price\n",
      "Explanatory variable set: X_3\n",
      "Number of observations: 34344/43383\n",
      "OLS MSE: 762.7433172415257\n",
      "Lasso MSE: 772.6439722126386\n",
      "Ridge MSE: 762.7441271491298\n",
      "Random Forest MSE: 254.85252480178346\n",
      "Gradient Boosting MSE: 262.1928849618694\n",
      "\n",
      "\n",
      "Model 4\n",
      "Outcome variable: trade_price_log\n",
      "Explanatory variable set: X_1\n",
      "Number of observations: 34344/43383\n",
      "OLS MSE: 0.9652796255375111\n",
      "Lasso MSE: 2.0694651928036882\n",
      "Ridge MSE: 0.9652760163474333\n",
      "Random Forest MSE: 0.20086447161341292\n",
      "Gradient Boosting MSE: 0.22030087195595155\n",
      "\n",
      "\n",
      "Model 5\n",
      "Outcome variable: trade_price_log\n",
      "Explanatory variable set: X_2\n",
      "Number of observations: 43383/43383\n",
      "OLS MSE: 1.6298153106926545\n",
      "Lasso MSE: 2.726420550065845\n",
      "Ridge MSE: 1.6298206621505174\n",
      "Random Forest MSE: 0.42848719809965813\n",
      "Gradient Boosting MSE: 0.42823943784356744\n",
      "\n",
      "\n",
      "Model 6\n",
      "Outcome variable: trade_price_log\n",
      "Explanatory variable set: X_3\n",
      "Number of observations: 34344/43383\n",
      "OLS MSE: 1.5219998378514938\n",
      "Lasso MSE: 2.728343636149774\n",
      "Ridge MSE: 1.5220032349272732\n",
      "Random Forest MSE: 0.2669527802567045\n",
      "Gradient Boosting MSE: 0.28226379151180725\n",
      "\n",
      "\n",
      "Model 7\n",
      "Outcome variable: amount_usd\n",
      "Explanatory variable set: X_1\n",
      "Number of observations: 34344/43383\n",
      "OLS MSE: 5014269850.1320305\n",
      "Lasso MSE: 5014270920.24065\n",
      "Ridge MSE: 5014251138.690217\n",
      "Random Forest MSE: 1294238452.6700416\n",
      "Gradient Boosting MSE: 2030209761.3251388\n",
      "\n",
      "\n",
      "Model 8\n",
      "Outcome variable: amount_usd\n",
      "Explanatory variable set: X_2\n",
      "Number of observations: 43383/43383\n",
      "OLS MSE: 4366992668.15671\n",
      "Lasso MSE: 4366976902.222889\n",
      "Ridge MSE: 4367009547.694157\n",
      "Random Forest MSE: 1268160732.9657876\n",
      "Gradient Boosting MSE: 2291411007.843155\n",
      "\n",
      "\n",
      "Model 9\n",
      "Outcome variable: amount_usd\n",
      "Explanatory variable set: X_3\n",
      "Number of observations: 34344/43383\n",
      "OLS MSE: 8291951737.812925\n",
      "Lasso MSE: 8291930656.153415\n",
      "Ridge MSE: 8291953302.324869\n",
      "Random Forest MSE: 2909595459.231181\n",
      "Gradient Boosting MSE: 3234755458.7974563\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i, outcome in enumerate([y_col_1, y_col_2, y_col_3]):\n",
    "    for j, explanatory in enumerate([X_cols_1, X_cols_2, X_cols_3]):\n",
    "        print(f\"Model {i*3+j+1}\")\n",
    "        print(f\"Outcome variable: {outcome}\")\n",
    "        print(f\"Explanatory variable set: X_{j+1}\")\n",
    "        benchmark_models(df, X_cols=explanatory, y_col=outcome)\n",
    "        print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the benchmarking results, we can see that inclusion of last_trade variables yields positive performance gains while maintaining 80% of all training data.  \n",
    "Hence in the next section, we will use the full set of features to train the deep neural network."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}