examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to XGBoost-Spark Cross Validation with GPU\n",
    "\n",
    "The goal of this notebook is to show you how to levarage GPU to accelerate XGBoost spark cross validatoin for hyperparameter tuning. The best model for the given hyperparameters will be returned.\n",
    "\n",
    "Here takes the application 'Mortgage' as an example.\n",
    "\n",
    "A few libraries are required for this notebook:\n",
    "  1. cudf-cu11\n",
    "  2. xgboost\n",
    "  3. scikit-learn\n",
    "  4. numpy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import the Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost.spark import SparkXGBClassifier, SparkXGBClassifierModel\n",
    "from pyspark.ml.tuning import ParamGridBuilder, CrossValidator\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import FloatType, IntegerType, StructField, StructType, DoubleType\n",
    "from pyspark.conf import SparkConf\n",
    "from time import time\n",
    "import os\n",
    "# if you pass/unpack the archive file and enable the environment\n",
    "# os.environ['PYSPARK_PYTHON'] = \"./environment/bin/python\"\n",
    "# os.environ['PYSPARK_DRIVER_PYTHON'] = \"./environment/bin/python\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a Spark Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-25 09:34:43,524 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "2022-11-25 09:34:43,952 WARN resource.ResourceUtils: The configuration of cores (exec = 4 task = 1, runnable tasks = 4) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n",
      "2022-11-25 09:34:58,155 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.06.0 using cudf 24.06.0.\n",
      "2022-11-25 09:34:58,171 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n",
      "2022-11-25 09:34:58,175 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n",
      "2022-11-25 09:34:58,175 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n"
     ]
    }
   ],
   "source": [
    "SPARK_MASTER_URL = os.getenv(\"SPARK_MASTER_URL\", \"/your-url\")\n",
    "\n",
    "RAPIDS_JAR = os.getenv(\"RAPIDS_JAR\", \"/your-jar-path\")\n",
    "\n",
    "# You need to update with your real hardware resource \n",
    "driverMem = os.getenv(\"DRIVER_MEM\", \"2g\")\n",
    "executorMem = os.getenv(\"EXECUTOR_MEM\", \"2g\")\n",
    "pinnedPoolSize = os.getenv(\"PINNED_POOL_SIZE\", \"2g\")\n",
    "concurrentGpuTasks = os.getenv(\"CONCURRENT_GPU_TASKS\", \"2\")\n",
    "executorCores = int(os.getenv(\"EXECUTOR_CORES\", \"4\"))\n",
    "# Common spark settings\n",
    "conf = SparkConf()\n",
    "conf.setMaster(SPARK_MASTER_URL)\n",
    "conf.setAppName(\"Microbenchmark on GPU\")\n",
    "conf.set(\"spark.driver.memory\", driverMem)\n",
    "## The tasks will run on GPU memory, so there is no need to set a high host memory\n",
    "conf.set(\"spark.executor.memory\", executorMem)\n",
    "## The tasks will run on GPU cores, so there is no need to use many cpu cores\n",
    "conf.set(\"spark.executor.cores\", executorCores)\n",
    "\n",
    "# Plugin settings\n",
    "conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n",
    "conf.set(\"spark.rapids.sql.concurrentGpuTasks\", concurrentGpuTasks)\n",
    "conf.set(\"spark.rapids.memory.pinnedPool.size\", pinnedPoolSize)\n",
    "conf.set(\"spark.rapids.memory.gpu.allocFraction\",\"0.7\")\n",
    "conf.set(\"spark.locality.wait\",\"0\")\n",
    "##############note: only support value=1 https://github.com/dmlc/xgboost/blame/master/python-package/xgboost/spark/core.py#L370-L374\n",
    "conf.set(\"spark.task.resource.gpu.amount\", 1) \n",
    "conf.set(\"spark.rapids.sql.enabled\", \"true\") \n",
    "conf.set(\"spark.plugins\", \"com.nvidia.spark.SQLPlugin\")\n",
    "conf.set(\"spark.sql.cache.serializer\",\"com.nvidia.spark.ParquetCachedBatchSerializer\")\n",
    "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", 200000) \n",
    "conf.set(\"spark.driver.extraClassPath\", RAPIDS_JAR)\n",
    "conf.set(\"spark.executor.extraClassPath\", RAPIDS_JAR)\n",
    "# if you pass/unpack the archive file and enable the environment\n",
    "# conf.set(\"spark.yarn.dist.archives\", \"your_pyspark_venv.tar.gz#environment\")\n",
    "# Create spark session\n",
    "spark = SparkSession.builder.config(conf=conf).getOrCreate()\n",
    "\n",
    "reader = spark.read"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Specify the Data Schema and Load the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 'delinquency_12'\n",
    "schema = StructType([\n",
    "    StructField('orig_channel', FloatType()),\n",
    "    StructField('first_home_buyer', FloatType()),\n",
    "    StructField('loan_purpose', FloatType()),\n",
    "    StructField('property_type', FloatType()),\n",
    "    StructField('occupancy_status', FloatType()),\n",
    "    StructField('property_state', FloatType()),\n",
    "    StructField('product_type', FloatType()),\n",
    "    StructField('relocation_mortgage_indicator', FloatType()),\n",
    "    StructField('seller_name', FloatType()),\n",
    "    StructField('mod_flag', FloatType()),\n",
    "    StructField('orig_interest_rate', FloatType()),\n",
    "    StructField('orig_upb', DoubleType()),\n",
    "    StructField('orig_loan_term', IntegerType()),\n",
    "    StructField('orig_ltv', FloatType()),\n",
    "    StructField('orig_cltv', FloatType()),\n",
    "    StructField('num_borrowers', FloatType()),\n",
    "    StructField('dti', FloatType()),\n",
    "    StructField('borrower_credit_score', FloatType()),\n",
    "    StructField('num_units', IntegerType()),\n",
    "    StructField('zip', IntegerType()),\n",
    "    StructField('mortgage_insurance_percent', FloatType()),\n",
    "    StructField('current_loan_delinquency_status', IntegerType()),\n",
    "    StructField('current_actual_upb', FloatType()),\n",
    "    StructField('interest_rate', FloatType()),\n",
    "    StructField('loan_age', FloatType()),\n",
    "    StructField('msa', FloatType()),\n",
    "    StructField('non_interest_bearing_upb', FloatType()),\n",
    "    StructField(label, IntegerType()),\n",
    "])\n",
    "features = [ x.name for x in schema if x.name != label ]\n",
    "\n",
    "# You need to update them to your real paths!\n",
    "dataRoot = os.getenv(\"DATA_ROOT\", \"/data\")\n",
    "train_path = dataRoot + \"/mortgage/output/train\"\n",
    "eval_path = dataRoot + \"/mortgage/output/eval\"\n",
    "\n",
    "data_format = 'parquet'\n",
    "has_header = 'true'\n",
    "if data_format == 'csv':\n",
    "    train_data = reader.schema(schema).option('header',has_header).csv(train_path)\n",
    "    trans_data = reader.schema(schema).option('header',has_header).csv(eval_path)\n",
    "else :\n",
    "    train_data = reader.load(train_path)\n",
    "    trans_data = reader.load(eval_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build a XGBoost-Spark CrossValidator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = { \n",
    "    \"tree_method\": \"gpu_hist\",\n",
    "    \"grow_policy\": \"depthwise\",\n",
    "    \"num_workers\": 1,\n",
    "    \"device\": \"cuda\",\n",
    "}\n",
    "\n",
    "params['features_col'] = features\n",
    "params['label_col'] = label\n",
    "    \n",
    "classifier = SparkXGBClassifier(**params)\n",
    "\n",
    "# Then build the evaluator and the hyperparameters\n",
    "evaluator = (MulticlassClassificationEvaluator()\n",
    "    .setLabelCol(label))\n",
    "param_grid = (ParamGridBuilder()\n",
    "    .addGrid(classifier.max_depth, [3, 6])\n",
    "    .addGrid(classifier.n_estimators, [100, 200])\n",
    "    .build())\n",
    "# Finally the corss validator\n",
    "cross_validator = (CrossValidator()\n",
    "    .setEstimator(classifier)\n",
    "    .setEvaluator(evaluator)\n",
    "    .setEstimatorParamMaps(param_grid)\n",
    "    .setNumFolds(2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Start Cross Validation by Fitting Data to CrossValidator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-25 09:35:01,049 WARN util.package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n",
      "If features_cols param set, then features_col param is ignored.\n",
      "/data/home/yuanli/work/reviews/pr252/pyspark_venv_20221125/lib/python3.8/site-packages/xgboost/sklearn.py:808: UserWarning: Loading a native XGBoost model with Scikit-Learn interface.\n",
      "  warnings.warn(\"Loading a native XGBoost model with Scikit-Learn interface.\")\n",
      "2022-11-25 09:35:26,758 WARN rapids.GpuOverrides: \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#2153, delinquency_12#2255, 1.0#2256, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#2153 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#2255 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#2256 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#2186 cannot run on GPU because expression AttributeReference probability#2186 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#2261 cannot run on GPU because expression AttributeReference obj#2261 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because not all expressions can be replaced; unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#2186]\n",
      "    @Expression <Alias> pythonUDF0#2552.prediction AS prediction#2153 could run on GPU\n",
      "      @Expression <GetStructField> pythonUDF0#2552.prediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#2552 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#27 as double) AS delinquency_12#2255 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#27 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#27 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#2256 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <Alias> UDF(pythonUDF0#2552.probability) AS probability#2186 cannot run on GPU because input expression ScalaUDF UDF(pythonUDF0#2552.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported); expression Alias UDF(pythonUDF0#2552.probability) AS probability#2186 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "      !Expression <ScalaUDF> UDF(pythonUDF0#2552.probability) cannot run on GPU because expression ScalaUDF UDF(pythonUDF0#2552.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled\n",
      "        @Expression <GetStructField> pythonUDF0#2552.probability could run on GPU\n",
      "          @Expression <AttributeReference> pythonUDF0#2552 could run on GPU\n",
      "\n",
      "If features_cols param set, then features_col param is ignored.                 \n",
      "2022-11-25 09:35:34,074 WARN rapids.GpuOverrides:                               \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#4415, delinquency_12#4517, 1.0#4518, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#4415 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#4517 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#4518 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#4448 cannot run on GPU because expression AttributeReference probability#4448 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#4523 cannot run on GPU because expression AttributeReference obj#4523 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#4448]; not all expressions can be replaced\n",
      "    @Expression <Alias> pythonUDF0#4814.prediction AS prediction#4415 could run on GPU\n",
      "      @Expression <GetStructField> pythonUDF0#4814.prediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#4814 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#27 as double) AS delinquency_12#4517 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#27 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#27 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#4518 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <Alias> UDF(pythonUDF0#4814.probability) AS probability#4448 cannot run on GPU because expression Alias UDF(pythonUDF0#4814.probability) AS probability#4448 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; input expression ScalaUDF UDF(pythonUDF0#4814.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported)\n",
      "      !Expression <ScalaUDF> UDF(pythonUDF0#4814.probability) cannot run on GPU because neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled; expression ScalaUDF UDF(pythonUDF0#4814.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "        @Expression <GetStructField> pythonUDF0#4814.probability could run on GPU\n",
      "          @Expression <AttributeReference> pythonUDF0#4814 could run on GPU\n",
      "\n",
      "If features_cols param set, then features_col param is ignored.\n",
      "2022-11-25 09:35:37,859 WARN rapids.GpuOverrides:                               \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#6677, delinquency_12#6779, 1.0#6780, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#6677 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#6779 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#6780 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#6710 cannot run on GPU because expression AttributeReference probability#6710 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#6785 cannot run on GPU because expression AttributeReference obj#6785 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#6710]; not all expressions can be replaced\n",
      "    @Expression <Alias> pythonUDF0#7076.prediction AS prediction#6677 could run on GPU\n",
      "      @Expression <GetStructField> pythonUDF0#7076.prediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#7076 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#27 as double) AS delinquency_12#6779 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#27 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#27 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#6780 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <Alias> UDF(pythonUDF0#7076.probability) AS probability#6710 cannot run on GPU because input expression ScalaUDF UDF(pythonUDF0#7076.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported); expression Alias UDF(pythonUDF0#7076.probability) AS probability#6710 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "      !Expression <ScalaUDF> UDF(pythonUDF0#7076.probability) cannot run on GPU because expression ScalaUDF UDF(pythonUDF0#7076.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled\n",
      "        @Expression <GetStructField> pythonUDF0#7076.probability could run on GPU\n",
      "          @Expression <AttributeReference> pythonUDF0#7076 could run on GPU\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "If features_cols param set, then features_col param is ignored.\n",
      "2022-11-25 09:35:41,551 WARN rapids.GpuOverrides:                               \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#8939, delinquency_12#9041, 1.0#9042, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#8939 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#9041 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#9042 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#8972 cannot run on GPU because expression AttributeReference probability#8972 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#9047 cannot run on GPU because expression AttributeReference obj#9047 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#8972]; not all expressions can be replaced\n",
      "    @Expression <Alias> pythonUDF0#9338.prediction AS prediction#8939 could run on GPU\n",
      "      @Expression <GetStructField> pythonUDF0#9338.prediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#9338 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#27 as double) AS delinquency_12#9041 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#27 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#27 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#9042 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <Alias> UDF(pythonUDF0#9338.probability) AS probability#8972 cannot run on GPU because input expression ScalaUDF UDF(pythonUDF0#9338.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported); expression Alias UDF(pythonUDF0#9338.probability) AS probability#8972 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "      !Expression <ScalaUDF> UDF(pythonUDF0#9338.probability) cannot run on GPU because neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled; expression ScalaUDF UDF(pythonUDF0#9338.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "        @Expression <GetStructField> pythonUDF0#9338.probability could run on GPU\n",
      "          @Expression <AttributeReference> pythonUDF0#9338 could run on GPU\n",
      "\n",
      "If features_cols param set, then features_col param is ignored.\n",
      "2022-11-25 09:35:45,231 WARN rapids.GpuOverrides:                               \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#11491, delinquency_12#11593, 1.0#11594, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#11491 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#11593 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#11594 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#11524 cannot run on GPU because expression AttributeReference probability#11524 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#11599 cannot run on GPU because expression AttributeReference obj#11599 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because not all expressions can be replaced; unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#11524]\n",
      "    @Expression <Alias> pythonUDF0#11890.prediction AS prediction#11491 could run on GPU\n",
      "      @Expression <GetStructField> pythonUDF0#11890.prediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#11890 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#27 as double) AS delinquency_12#11593 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#27 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#27 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#11594 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <Alias> UDF(pythonUDF0#11890.probability) AS probability#11524 cannot run on GPU because expression Alias UDF(pythonUDF0#11890.probability) AS probability#11524 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; input expression ScalaUDF UDF(pythonUDF0#11890.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported)\n",
      "      !Expression <ScalaUDF> UDF(pythonUDF0#11890.probability) cannot run on GPU because neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled; expression ScalaUDF UDF(pythonUDF0#11890.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "        @Expression <GetStructField> pythonUDF0#11890.probability could run on GPU\n",
      "          @Expression <AttributeReference> pythonUDF0#11890 could run on GPU\n",
      "\n",
      "If features_cols param set, then features_col param is ignored.\n",
      "2022-11-25 09:35:49,003 WARN rapids.GpuOverrides:                               \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#13753, delinquency_12#13855, 1.0#13856, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#13753 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#13855 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#13856 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#13786 cannot run on GPU because expression AttributeReference probability#13786 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#13861 cannot run on GPU because expression AttributeReference obj#13861 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#13786]; not all expressions can be replaced\n",
      "    @Expression <Alias> pythonUDF0#14152.prediction AS prediction#13753 could run on GPU\n",
      "      @Expression <GetStructField> pythonUDF0#14152.prediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#14152 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#27 as double) AS delinquency_12#13855 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#27 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#27 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#13856 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <Alias> UDF(pythonUDF0#14152.probability) AS probability#13786 cannot run on GPU because expression Alias UDF(pythonUDF0#14152.probability) AS probability#13786 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; input expression ScalaUDF UDF(pythonUDF0#14152.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported)\n",
      "      !Expression <ScalaUDF> UDF(pythonUDF0#14152.probability) cannot run on GPU because expression ScalaUDF UDF(pythonUDF0#14152.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled\n",
      "        @Expression <GetStructField> pythonUDF0#14152.probability could run on GPU\n",
      "          @Expression <AttributeReference> pythonUDF0#14152 could run on GPU\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "If features_cols param set, then features_col param is ignored.\n",
      "2022-11-25 09:35:52,578 WARN rapids.GpuOverrides:                               \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#16015, delinquency_12#16117, 1.0#16118, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#16015 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#16117 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#16118 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#16048 cannot run on GPU because expression AttributeReference probability#16048 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#16123 cannot run on GPU because expression AttributeReference obj#16123 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#16048]; not all expressions can be replaced\n",
      "    @Expression <Alias> pythonUDF0#16414.prediction AS prediction#16015 could run on GPU\n",
      "      @Expression <GetStructField> pythonUDF0#16414.prediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#16414 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#27 as double) AS delinquency_12#16117 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#27 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#27 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#16118 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <Alias> UDF(pythonUDF0#16414.probability) AS probability#16048 cannot run on GPU because expression Alias UDF(pythonUDF0#16414.probability) AS probability#16048 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; input expression ScalaUDF UDF(pythonUDF0#16414.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported)\n",
      "      !Expression <ScalaUDF> UDF(pythonUDF0#16414.probability) cannot run on GPU because expression ScalaUDF UDF(pythonUDF0#16414.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled\n",
      "        @Expression <GetStructField> pythonUDF0#16414.probability could run on GPU\n",
      "          @Expression <AttributeReference> pythonUDF0#16414 could run on GPU\n",
      "\n",
      "If features_cols param set, then features_col param is ignored.\n",
      "2022-11-25 09:35:56,267 WARN rapids.GpuOverrides:                               \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#18277, delinquency_12#18379, 1.0#18380, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#18277 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#18379 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#18380 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#18310 cannot run on GPU because expression AttributeReference probability#18310 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#18385 cannot run on GPU because expression AttributeReference obj#18385 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#18310]; not all expressions can be replaced\n",
      "    @Expression <Alias> pythonUDF0#18676.prediction AS prediction#18277 could run on GPU\n",
      "      @Expression <GetStructField> pythonUDF0#18676.prediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#18676 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#27 as double) AS delinquency_12#18379 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#27 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#27 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#18380 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <Alias> UDF(pythonUDF0#18676.probability) AS probability#18310 cannot run on GPU because expression Alias UDF(pythonUDF0#18676.probability) AS probability#18310 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; input expression ScalaUDF UDF(pythonUDF0#18676.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported)\n",
      "      !Expression <ScalaUDF> UDF(pythonUDF0#18676.probability) cannot run on GPU because neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled; expression ScalaUDF UDF(pythonUDF0#18676.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "        @Expression <GetStructField> pythonUDF0#18676.probability could run on GPU\n",
      "          @Expression <AttributeReference> pythonUDF0#18676 could run on GPU\n",
      "\n",
      "If features_cols param set, then features_col param is ignored.\n",
      "[Stage 69:>                                                         (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Validation takes 59.46 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "def with_benchmark(phrase, action):\n",
    "    start = time()\n",
    "    result = action()\n",
    "    end = time()\n",
    "    print('{} takes {} seconds'.format(phrase, round(end - start, 2)))\n",
    "    return result\n",
    "model = with_benchmark('Cross-Validation', lambda: cross_validator.fit(train_data)).bestModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transform On the Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-25 09:35:59,886 WARN rapids.GpuOverrides: \n",
      "!Exec <ProjectExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [rawPrediction#18908, probability#18974]; not all expressions can be replaced\n",
      "  @Expression <AttributeReference> orig_channel#56 could run on GPU\n",
      "  @Expression <AttributeReference> first_home_buyer#57 could run on GPU\n",
      "  @Expression <AttributeReference> loan_purpose#58 could run on GPU\n",
      "  @Expression <AttributeReference> property_type#59 could run on GPU\n",
      "  @Expression <AttributeReference> occupancy_status#60 could run on GPU\n",
      "  @Expression <AttributeReference> property_state#61 could run on GPU\n",
      "  @Expression <AttributeReference> product_type#62 could run on GPU\n",
      "  @Expression <AttributeReference> relocation_mortgage_indicator#63 could run on GPU\n",
      "  @Expression <AttributeReference> seller_name#64 could run on GPU\n",
      "  @Expression <AttributeReference> mod_flag#65 could run on GPU\n",
      "  @Expression <AttributeReference> orig_interest_rate#66 could run on GPU\n",
      "  @Expression <AttributeReference> orig_upb#67 could run on GPU\n",
      "  @Expression <AttributeReference> orig_loan_term#68 could run on GPU\n",
      "  @Expression <AttributeReference> orig_ltv#69 could run on GPU\n",
      "  @Expression <AttributeReference> orig_cltv#70 could run on GPU\n",
      "  @Expression <AttributeReference> num_borrowers#71 could run on GPU\n",
      "  @Expression <AttributeReference> dti#72 could run on GPU\n",
      "  @Expression <AttributeReference> borrower_credit_score#73 could run on GPU\n",
      "  @Expression <AttributeReference> num_units#74 could run on GPU\n",
      "  @Expression <AttributeReference> zip#75 could run on GPU\n",
      "  @Expression <AttributeReference> mortgage_insurance_percent#76 could run on GPU\n",
      "  @Expression <AttributeReference> current_loan_delinquency_status#77 could run on GPU\n",
      "  @Expression <AttributeReference> current_actual_upb#78 could run on GPU\n",
      "  @Expression <AttributeReference> interest_rate#79 could run on GPU\n",
      "  @Expression <AttributeReference> loan_age#80 could run on GPU\n",
      "  @Expression <AttributeReference> msa#81 could run on GPU\n",
      "  @Expression <AttributeReference> non_interest_bearing_upb#82 could run on GPU\n",
      "  @Expression <AttributeReference> delinquency_12#83 could run on GPU\n",
      "  !Expression <Alias> UDF(pythonUDF0#19041.rawPrediction) AS rawPrediction#18908 cannot run on GPU because input expression ScalaUDF UDF(pythonUDF0#19041.rawPrediction) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported); expression Alias UDF(pythonUDF0#19041.rawPrediction) AS rawPrediction#18908 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "    !Expression <ScalaUDF> UDF(pythonUDF0#19041.rawPrediction) cannot run on GPU because neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled; expression ScalaUDF UDF(pythonUDF0#19041.rawPrediction) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "      @Expression <GetStructField> pythonUDF0#19041.rawPrediction could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#19041 could run on GPU\n",
      "  @Expression <Alias> pythonUDF0#19041.prediction AS prediction#18942 could run on GPU\n",
      "    @Expression <GetStructField> pythonUDF0#19041.prediction could run on GPU\n",
      "      @Expression <AttributeReference> pythonUDF0#19041 could run on GPU\n",
      "  !Expression <Alias> UDF(pythonUDF0#19041.probability) AS probability#18974 cannot run on GPU because expression Alias UDF(pythonUDF0#19041.probability) AS probability#18974 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7; input expression ScalaUDF UDF(pythonUDF0#19041.probability) (org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 is not supported)\n",
      "    !Expression <ScalaUDF> UDF(pythonUDF0#19041.probability) cannot run on GPU because neither UDF implemented by class org.apache.spark.ml.functions$$$Lambda$3606/1625633331 provides a GPU implementation, nor the conf `spark.rapids.sql.rowBasedUDF.enabled` is enabled; expression ScalaUDF UDF(pythonUDF0#19041.probability) produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "      @Expression <GetStructField> pythonUDF0#19041.probability could run on GPU\n",
      "        @Expression <AttributeReference> pythonUDF0#19041 could run on GPU\n",
      "\n",
      "2022-11-25 09:35:59,893 WARN rapids.GpuOverrides: \n",
      "!Exec <InMemoryTableScanExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [rawPrediction, probability]; unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [rawPrediction#18908, probability#18974]; not all expressions can be replaced\n",
      "  @Expression <AttributeReference> orig_channel#56 could run on GPU\n",
      "  @Expression <AttributeReference> first_home_buyer#57 could run on GPU\n",
      "  @Expression <AttributeReference> loan_purpose#58 could run on GPU\n",
      "  @Expression <AttributeReference> property_type#59 could run on GPU\n",
      "  @Expression <AttributeReference> occupancy_status#60 could run on GPU\n",
      "  @Expression <AttributeReference> property_state#61 could run on GPU\n",
      "  @Expression <AttributeReference> product_type#62 could run on GPU\n",
      "  @Expression <AttributeReference> relocation_mortgage_indicator#63 could run on GPU\n",
      "  @Expression <AttributeReference> seller_name#64 could run on GPU\n",
      "  @Expression <AttributeReference> mod_flag#65 could run on GPU\n",
      "  @Expression <AttributeReference> orig_interest_rate#66 could run on GPU\n",
      "  @Expression <AttributeReference> orig_upb#67 could run on GPU\n",
      "  @Expression <AttributeReference> orig_loan_term#68 could run on GPU\n",
      "  @Expression <AttributeReference> orig_ltv#69 could run on GPU\n",
      "  @Expression <AttributeReference> orig_cltv#70 could run on GPU\n",
      "  @Expression <AttributeReference> num_borrowers#71 could run on GPU\n",
      "  @Expression <AttributeReference> dti#72 could run on GPU\n",
      "  @Expression <AttributeReference> borrower_credit_score#73 could run on GPU\n",
      "  @Expression <AttributeReference> num_units#74 could run on GPU\n",
      "  @Expression <AttributeReference> zip#75 could run on GPU\n",
      "  @Expression <AttributeReference> mortgage_insurance_percent#76 could run on GPU\n",
      "  @Expression <AttributeReference> current_loan_delinquency_status#77 could run on GPU\n",
      "  @Expression <AttributeReference> current_actual_upb#78 could run on GPU\n",
      "  @Expression <AttributeReference> interest_rate#79 could run on GPU\n",
      "  @Expression <AttributeReference> loan_age#80 could run on GPU\n",
      "  @Expression <AttributeReference> msa#81 could run on GPU\n",
      "  @Expression <AttributeReference> non_interest_bearing_upb#82 could run on GPU\n",
      "  @Expression <AttributeReference> delinquency_12#83 could run on GPU\n",
      "  !Expression <AttributeReference> rawPrediction#18908 cannot run on GPU because expression AttributeReference rawPrediction#18908 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  @Expression <AttributeReference> prediction#18942 could run on GPU\n",
      "  !Expression <AttributeReference> probability#18974 cannot run on GPU because expression AttributeReference probability#18974 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "\n",
      "2022-11-25 09:36:00,975 WARN rapids.GpuOverrides:                               \n",
      "!Exec <CollectLimitExec> cannot run on GPU because the Exec CollectLimitExec has been disabled, and is disabled by default because Collect Limit replacement can be slower on the GPU, if huge number of rows in a batch it could help by limiting the number of rows transferred from GPU to CPU. Set spark.rapids.sql.exec.CollectLimitExec to true if you wish to enable it\n",
      "  @Partitioning <SinglePartition$> could run on GPU\n",
      "  !Exec <ProjectExec> cannot run on GPU because unsupported data types in input: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#18974, rawPrediction#18908]; not all expressions can be replaced\n",
      "    @Expression <Alias> cast(delinquency_12#83 as string) AS delinquency_12#19670 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#83 as string) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#83 could run on GPU\n",
      "    @Expression <Alias> cast(rawPrediction#18908 as string) AS rawPrediction#19671 could run on GPU\n",
      "      !Expression <Cast> cast(rawPrediction#18908 as string) cannot run on GPU because Cast from org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 to StringType is not supported\n",
      "        !Expression <AttributeReference> rawPrediction#18908 cannot run on GPU because expression AttributeReference rawPrediction#18908 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "    @Expression <Alias> cast(probability#18974 as string) AS probability#19672 could run on GPU\n",
      "      !Expression <Cast> cast(probability#18974 as string) cannot run on GPU because Cast from org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 to StringType is not supported\n",
      "        !Expression <AttributeReference> probability#18974 cannot run on GPU because expression AttributeReference probability#18974 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "    @Expression <Alias> cast(prediction#18942 as string) AS prediction#19673 could run on GPU\n",
      "      @Expression <Cast> cast(prediction#18942 as string) could run on GPU\n",
      "        @Expression <AttributeReference> prediction#18942 could run on GPU\n",
      "    !Exec <InMemoryTableScanExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#18974, rawPrediction#18908]; unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [rawPrediction, probability]; not all expressions can be replaced\n",
      "      @Expression <AttributeReference> delinquency_12#83 could run on GPU\n",
      "      @Expression <AttributeReference> prediction#18942 could run on GPU\n",
      "      !Expression <AttributeReference> probability#18974 cannot run on GPU because expression AttributeReference probability#18974 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "      !Expression <AttributeReference> rawPrediction#18908 cannot run on GPU because expression AttributeReference rawPrediction#18908 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transforming takes 1.15 seconds\n",
      "+--------------+--------------------+--------------------+----------+\n",
      "|delinquency_12|       rawPrediction|         probability|prediction|\n",
      "+--------------+--------------------+--------------------+----------+\n",
      "|             0|[10.2152490615844...|[0.99996340274810...|       0.0|\n",
      "|             0|[8.85215473175048...|[0.99985694885253...|       0.0|\n",
      "|             0|[8.85215473175048...|[0.99985694885253...|       0.0|\n",
      "|             0|[8.85215473175048...|[0.99985694885253...|       0.0|\n",
      "|             0|[10.2152490615844...|[0.99996340274810...|       0.0|\n",
      "+--------------+--------------------+--------------------+----------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def transform():\n",
    "    result = model.transform(trans_data).cache()\n",
    "    result.foreachPartition(lambda _: None)\n",
    "    return result\n",
    "result = with_benchmark('Transforming', transform)\n",
    "result.select(label, 'rawPrediction', 'probability', 'prediction').show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-25 09:36:01,155 WARN rapids.GpuOverrides: \n",
      "! <DeserializeToObjectExec> cannot run on GPU because not all expressions can be replaced; GPU does not currently support the operator class org.apache.spark.sql.execution.DeserializeToObjectExec\n",
      "  ! <CreateExternalRow> createexternalrow(prediction#18942, delinquency_12#20148, 1.0#20149, newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize, StructField(prediction,DoubleType,true), StructField(delinquency_12,DoubleType,true), StructField(1.0,DoubleType,false), StructField(probability,org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7,true)) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.CreateExternalRow\n",
      "    @Expression <AttributeReference> prediction#18942 could run on GPU\n",
      "    @Expression <AttributeReference> delinquency_12#20148 could run on GPU\n",
      "    @Expression <AttributeReference> 1.0#20149 could run on GPU\n",
      "    ! <Invoke> newInstance(class org.apache.spark.ml.linalg.VectorUDT).deserialize cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.Invoke\n",
      "      ! <NewInstance> newInstance(class org.apache.spark.ml.linalg.VectorUDT) cannot run on GPU because GPU does not currently support the operator class org.apache.spark.sql.catalyst.expressions.objects.NewInstance\n",
      "      !Expression <AttributeReference> probability#18974 cannot run on GPU because expression AttributeReference probability#18974 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "  !Expression <AttributeReference> obj#20154 cannot run on GPU because expression AttributeReference obj#20154 produces an unsupported type ObjectType(interface org.apache.spark.sql.Row)\n",
      "  !Exec <ProjectExec> cannot run on GPU because unsupported data types in input: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#18974]; not all expressions can be replaced; unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#18974]\n",
      "    @Expression <AttributeReference> prediction#18942 could run on GPU\n",
      "    @Expression <Alias> cast(delinquency_12#83 as double) AS delinquency_12#20148 could run on GPU\n",
      "      @Expression <Cast> cast(delinquency_12#83 as double) could run on GPU\n",
      "        @Expression <AttributeReference> delinquency_12#83 could run on GPU\n",
      "    @Expression <Alias> 1.0 AS 1.0#20149 could run on GPU\n",
      "      @Expression <Literal> 1.0 could run on GPU\n",
      "    !Expression <AttributeReference> probability#18974 cannot run on GPU because expression AttributeReference probability#18974 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "    !Exec <InMemoryTableScanExec> cannot run on GPU because unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [rawPrediction, probability]; not all expressions can be replaced; unsupported data types in output: org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7 [probability#18974]\n",
      "      @Expression <AttributeReference> delinquency_12#83 could run on GPU\n",
      "      @Expression <AttributeReference> prediction#18942 could run on GPU\n",
      "      !Expression <AttributeReference> probability#18974 cannot run on GPU because expression AttributeReference probability#18974 produces an unsupported type org.apache.spark.ml.linalg.VectorUDT@3bfc3ba7\n",
      "\n",
      "[Stage 72:>                                                         (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation takes 1.41 seconds\n",
      "Accuracy is 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "accuracy = with_benchmark(\n",
    "    'Evaluation',\n",
    "    lambda: MulticlassClassificationEvaluator().setLabelCol(label).evaluate(result))\n",
    "print('Accuracy is ' + str(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}