Skip to content

Commit

Permalink
More style changes
Browse files Browse the repository at this point in the history
  • Loading branch information
bvonodiripsa committed Aug 27, 2024
1 parent 50df271 commit b3820dc
Showing 1 changed file with 8 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@
" except Exception as e:\n",
" return str(e)\n",
"\n",
"\n",
"# Register the function as a UDF\n",
"extract_text_udf = udf(extract_text_from_binary_pdf, StringType())\n",
"\n",
Expand Down Expand Up @@ -328,7 +329,11 @@
"# Explode will distribute and replicate the content of a vecor across multple rows\n",
"# Add id column\n",
"\n",
"exploded_df = splitted_df.select(\"path\", explode(col(\"chunks\")).alias(\"chunk\")).select(\"path\", \"chunk\").withColumn(\"id\", monotonically_increasing_id())"
"exploded_df = (\n",
" splitted_df.select(\"path\", explode(col(\"chunks\")).alias(\"chunk\"))\n",
" .select(\"path\", \"chunk\")\n",
" .withColumn(\"id\", monotonically_increasing_id())\n",
")"
]
},
{
Expand Down Expand Up @@ -610,7 +615,7 @@
"result_df = (\n",
" knn_df.withColumn(\n",
" \"zipped\", F.explode(F.arrays_zip(F.col(\"indices\"), F.col(\"distances\")))\n",
" )\n",
" )\n",
" .select(\n",
" F.col(\"query_id\"),\n",
" F.col(\"zipped.indices\").alias(\"id\"),\n",
Expand Down Expand Up @@ -693,7 +698,7 @@
" print(\"Model is already loaded.\")\n",
"else:\n",
" print(\"Model is not loaded.\")\n",
" \n",
"\n",
" # Extend model input sizes\n",
" build_config = BuildConfig()\n",
" build_config.plugin_config.context_fmha = True\n",
Expand Down

0 comments on commit b3820dc

Please sign in to comment.