From 33180ef9a93f0da31a1b8fbdb6b892135665e79e Mon Sep 17 00:00:00 2001
From: Mark Hamilton <mhamilton723@gmail.com>
Date: Wed, 17 Jul 2024 14:39:51 -0400
Subject: [PATCH] chore: fix remaining build errors (#2250)

* chore: fix remaining build errors

* chore: fix remaining build errors

* chore: fix remaining build errors

* chore: fix remaining build errors
---
 .../synapse/ml/io/http/RESTHelpers.scala      |  9 ++++++---
 .../ml/nbtest/DatabricksUtilities.scala       |  9 +++++++--
 ...tart - Fine-tune a Vision Classifier.ipynb |  1 -
 pipeline.yaml                                 | 20 +++++++++----------
 4 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/RESTHelpers.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/RESTHelpers.scala
index eb6ac62fea..d5f4ea1bf4 100644
--- a/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/RESTHelpers.scala
+++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/RESTHelpers.scala
@@ -50,7 +50,7 @@ object RESTHelpers {
   }
 
   def safeSend(request: HttpRequestBase,
-               backoffs: List[Int] = List(100, 500, 1000),  //scalastyle:ignore magic.number
+               backoffs: List[Int] = List(100, 500, 1000), //scalastyle:ignore magic.number
                expectedCodes: Set[Int] = Set(),
                close: Boolean = true): CloseableHttpResponse = {
 
@@ -92,8 +92,11 @@ object RESTHelpers {
     IOUtils.toString(result.getEntity.getContent, "utf-8")
   }
 
-  def sendAndParseJson(request: HttpRequestBase, expectedCodes: Set[Int]=Set()): JsValue = {
-    val response = safeSend(request, expectedCodes=expectedCodes, close=false)
+  def sendAndParseJson(request: HttpRequestBase,
+                       expectedCodes: Set[Int] = Set(),
+                       backoffs: List[Int] = List(100, 500, 1000) //scalastyle:ignore magic.number
+                      ): JsValue = {
+    val response = safeSend(request, expectedCodes = expectedCodes, close = false, backoffs = backoffs)
     val output = parseResult(response).parseJson
     response.close()
     output
diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala
index 4eac2c5de1..8a012918e9 100644
--- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala
+++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala
@@ -25,6 +25,7 @@ import scala.collection.immutable.Map
 import scala.collection.mutable
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, ExecutionContext, Future, blocking}
+import scala.util.Random
 
 object DatabricksUtilities {
 
@@ -116,7 +117,10 @@ object DatabricksUtilities {
   def databricksGet(path: String, apiVersion: String = "2.0"): JsValue = {
     val request = new HttpGet(baseURL(apiVersion) + path)
     request.addHeader("Authorization", AuthValue)
-    RESTHelpers.sendAndParseJson(request)
+    val random = new Random() // Use a jittered retry to avoid overwhelming
+    RESTHelpers.sendAndParseJson(request, backoffs = List.fill(3) {
+      1000 + random.nextInt(1000)
+    })
   }
 
   //TODO convert all this to typed code
@@ -332,7 +336,8 @@ object DatabricksUtilities {
       lifeCycleState = lcs
       if (logLevel >= 2) println(s"Job $runId state: $lifeCycleState")
       blocking {
-        Thread.sleep(interval.toLong)
+        val random = new Random() // Use a jittered retry to avoid overwhelming
+        Thread.sleep(interval.toLong + random.nextInt(1000))
       }
     }
 
diff --git a/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.ipynb b/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.ipynb
index 54ef948c34..8bb100593e 100644
--- a/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.ipynb	
+++ b/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.ipynb	
@@ -111,7 +111,6 @@
     "    spark.read.format(\"binaryFile\")\n",
     "    .option(\"pathGlobFilter\", \"*.jpg\")\n",
     "    .load(folder_path + \"/train\")\n",
-    "    .sample(0.5)  # For demo purposes\n",
     "    .withColumn(\"image\", regexp_replace(\"path\", \"dbfs:\", \"/dbfs\"))\n",
     "    .withColumn(\"label\", assign_label_udf(col(\"path\")))\n",
     "    .select(\"image\", \"label\")\n",
diff --git a/pipeline.yaml b/pipeline.yaml
index 1d93bc7919..816627a134 100644
--- a/pipeline.yaml
+++ b/pipeline.yaml
@@ -116,19 +116,17 @@ jobs:
         PGP-PUBLIC: $(pgp-public)
         PGP-PW: $(pgp-pw)
         SYNAPSEML_ENABLE_PUBLISH: true
-    - bash: |
-        set -e
-        sbt publishBadges
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: 'SynapseML Build'
+        scriptLocation: inlineScript
+        scriptType: bash
+        inlineScript: |
+          set -e
+          sbt publishBadges
       condition: and(succeeded(), eq(variables.isMaster, true))
       displayName: Publish Badges
-      env:
-        STORAGE-KEY: $(storage-key)
-        NEXUS-UN: $(nexus-un)
-        NEXUS-PW: $(nexus-pw)
-        PGP-PRIVATE: $(pgp-private)
-        PGP-PUBLIC: $(pgp-public)
-        PGP-PW: $(pgp-pw)
-        SYNAPSEML_ENABLE_PUBLISH: true
+
 
 - job: E2E
   timeoutInMinutes: 120