chore: fix some of the failing build issues (#2071)

microsoft · Sep 13, 2023 · dd7ba7c · dd7ba7c
1 parent 3d1a7fc
commit dd7ba7c
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 3 deletions.
diff --git a/build.sbt b/build.sbt
@@ -25,6 +25,7 @@ val coreDependencies = Seq(
   "org.apache.spark" %% "spark-mllib" % sparkVersion % "compile",
   "org.apache.spark" %% "spark-avro" % sparkVersion % "provided",
   "org.apache.spark" %% "spark-tags" % sparkVersion % "test",
+  "com.globalmentor" % "hadoop-bare-naked-local-fs" % "0.1.0" % "test",
   "org.scalatest" %% "scalatest" % "3.2.14" % "test")
 val extraDependencies = Seq(
   "org.scalactic" %% "scalactic" % "3.2.14",

diff --git a/.../com/microsoft/azure/synapse/ml/cognitive/anomaly/MultivariateAnamolyDetectionSuite.scala b/.../com/microsoft/azure/synapse/ml/cognitive/anomaly/MultivariateAnamolyDetectionSuite.scala
@@ -4,7 +4,7 @@
 package com.microsoft.azure.synapse.ml.cognitive.anomaly
 
 import com.microsoft.azure.synapse.ml.Secrets
-import com.microsoft.azure.synapse.ml.core.test.base.TestBase
+import com.microsoft.azure.synapse.ml.core.test.base.{Flaky, TestBase}
 import com.microsoft.azure.synapse.ml.core.test.benchmarks.DatasetUtils
 import com.microsoft.azure.synapse.ml.core.test.fuzzing.{EstimatorFuzzing, TestObject, TransformerFuzzing}
 import org.apache.hadoop.conf.Configuration
@@ -62,7 +62,8 @@ trait MADTestUtils extends TestBase with AnomalyKey with StorageCredentials {
 
 }
 
-class SimpleFitMultivariateAnomalySuite extends EstimatorFuzzing[SimpleFitMultivariateAnomaly] with MADTestUtils {
+class SimpleFitMultivariateAnomalySuite extends EstimatorFuzzing[SimpleFitMultivariateAnomaly]
+  with MADTestUtils with Flaky {
 
   def simpleMultiAnomalyEstimator: SimpleFitMultivariateAnomaly = new SimpleFitMultivariateAnomaly()
     .setSubscriptionKey(anomalyKey)

diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/core/test/base/TestBase.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/core/test/base/TestBase.scala
@@ -6,7 +6,9 @@ package com.microsoft.azure.synapse.ml.core.test.base
 import breeze.linalg.norm.Impl
 import breeze.linalg.{*, norm, DenseMatrix => BDM, DenseVector => BDV}
 import breeze.math.Field
+import com.globalmentor.apache.hadoop.fs.BareLocalFileSystem
 import org.apache.commons.io.FileUtils
+import org.apache.hadoop.fs.FileSystem
 import org.apache.spark._
 import org.apache.spark.ml._
 import org.apache.spark.sql.{DataFrame, _}
@@ -67,6 +69,8 @@ trait SparkSessionManagement {
       .config(sparkConfiguration)
       .getOrCreate()
     sess.sparkContext.setLogLevel(logLevel)
+    sess.sparkContext.hadoopConfiguration
+      .setClass("fs.file.impl", classOf[BareLocalFileSystem], classOf[FileSystem])
     sess
   }
 

diff --git a/pipeline.yaml b/pipeline.yaml
@@ -93,7 +93,6 @@ jobs:
         source activate synapseml
         sbt packagePython
         sbt publishBlob publishDocs publishR publishPython uploadNotebooks
-        sbt publishSigned
         sbt genBuildInfo
         echo "##vso[task.uploadsummary]$(pwd)/target/Build.md"
       displayName: Publish Artifacts