Merge branch 'master' into sequence-should-support-update

src/main/scala/cognite/spark/v1/AssetsRelation.scala

@@ -122,13 +122,14 @@ object AssetsRelation
     with NamedRelation
     with InsertSchema
     with DeleteWithIdSchema
-    with UpdateSchemaFromUpsertSchema {
+    with UpdateSchema {
   override val name = "assets"
   import cognite.spark.compiletime.macros.StructTypeEncoderMacro._
 
   override val upsertSchema: StructType = structType[AssetsUpsertSchema]()
   override val insertSchema: StructType = structType[AssetsInsertSchema]()
   override val readSchema: StructType = structType[AssetsReadSchema]()
+  override val updateSchema: StructType = upsertSchema
 
 }
 

src/main/scala/cognite/spark/v1/DataSetsRelation.scala

@@ -73,14 +73,15 @@ object DataSetsRelation
     extends UpsertSchema
     with ReadSchema
     with InsertSchema
-    with UpdateSchemaFromUpsertSchema
+    with UpdateSchema
     with NamedRelation {
   override val name = "datasets"
   import cognite.spark.compiletime.macros.StructTypeEncoderMacro._
 
   override val upsertSchema: StructType = structType[DataSetsUpsertSchema]()
   override val insertSchema: StructType = structType[DataSetsInsertSchema]()
   override val readSchema: StructType = structType[DataSetsReadSchema]()
+  override val updateSchema: StructType = upsertSchema
 
 }
 

src/main/scala/cognite/spark/v1/EventsRelation.scala

@@ -94,13 +94,14 @@ object EventsRelation
     with NamedRelation
     with InsertSchema
     with DeleteWithIdSchema
-    with UpdateSchemaFromUpsertSchema {
+    with UpdateSchema {
   override val name: String = "events"
   import cognite.spark.compiletime.macros.StructTypeEncoderMacro._
 
   override val upsertSchema: StructType = structType[EventsUpsertSchema]()
   override val insertSchema: StructType = structType[EventsInsertSchema]()
   override val readSchema: StructType = structType[EventsReadSchema]()
+  override val updateSchema: StructType = upsertSchema
 }
 
 trait WithNullableExternalId extends WithExternalIdGeneric[OptionalField] {

src/main/scala/cognite/spark/v1/FileContentRelation.scala

@@ -46,9 +46,6 @@ class FileContentRelation(config: RelationConfig, fileExternalId: String, inferS
       )(backend => backend.close())
     } yield backend
 
-  val acceptedMimeTypes: Seq[String] =
-    Seq("application/jsonlines", "application/x-ndjson", "application/jsonl")
-
   @transient private lazy val dataFrame: DataFrame = createDataFrame
 
   override def schema: StructType =
@@ -64,15 +61,10 @@ class FileContentRelation(config: RelationConfig, fileExternalId: String, inferS
 
         val validUrl = for {
           file <- client.files.retrieveByExternalId(fileExternalId)
-          mimeType <- IO.pure(file.mimeType)
           _ <- IO.raiseWhen(!file.uploaded)(
             new CdfSparkException(
               f"Could not read file because no file was uploaded for externalId: $fileExternalId")
           )
-          _ <- IO.raiseWhen(mimeType.exists(!acceptedMimeTypes.contains(_)))(
-            new CdfSparkException("Wrong mimetype. Expects application/jsonlines")
-          )
-
           downloadLink <- client.files
             .downloadLink(FileDownloadExternalId(fileExternalId))
           uri <- IO.pure(uri"${downloadLink.downloadUrl}")

src/main/scala/cognite/spark/v1/FilesRelation.scala

@@ -115,15 +115,15 @@ object FilesRelation
     with ReadSchema
     with InsertSchema
     with DeleteWithIdSchema
-    with UpdateSchemaFromUpsertSchema
+    with UpdateSchema
     with NamedRelation {
   override val name: String = "files"
   import cognite.spark.compiletime.macros.StructTypeEncoderMacro._
 
   override val upsertSchema: StructType = structType[FilesUpsertSchema]()
   override val insertSchema: StructType = structType[FilesInsertSchema]()
   override val readSchema: StructType = structType[FilesReadSchema]()
-
+  override val updateSchema: StructType = upsertSchema
 }
 
 final case class FilesUpsertSchema(

src/main/scala/cognite/spark/v1/RelationSchemas.scala

@@ -30,10 +30,6 @@ trait UpdateSchema {
   val updateSchema: StructType
 }
 
-trait UpdateSchemaFromUpsertSchema extends UpsertSchema with UpdateSchema {
-  override val updateSchema: StructType = upsertSchema
-}
-
 // TODO: this isn't applied to some relations that have read support
 trait ReadSchema {
   val readSchema: StructType

src/main/scala/cognite/spark/v1/RelationshipsRelation.scala

@@ -143,14 +143,15 @@ object RelationshipsRelation
     with ReadSchema
     with DeleteWithExternalIdSchema
     with InsertSchema
-    with UpdateSchemaFromUpsertSchema
+    with UpdateSchema
     with NamedRelation {
   override val name: String = "relationships"
   import cognite.spark.compiletime.macros.StructTypeEncoderMacro._
 
   override val insertSchema: StructType = structType[RelationshipsInsertSchema]()
   override val readSchema: StructType = structType[RelationshipsReadSchema]()
   override val upsertSchema: StructType = structType[RelationshipsUpsertSchema]()
+  override val updateSchema: StructType = upsertSchema
 }
 
 final case class RelationshipsInsertSchema(

src/main/scala/cognite/spark/v1/TimeSeriesRelation.scala

@@ -3,7 +3,7 @@ package cognite.spark.v1
 import cats.effect.IO
 import cognite.spark.v1.PushdownUtilities._
 import cognite.spark.compiletime.macros.SparkSchemaHelper._
-import com.cognite.sdk.scala.common.{WithId}
+import com.cognite.sdk.scala.common.WithId
 import com.cognite.sdk.scala.v1._
 import com.cognite.sdk.scala.v1.resources.TimeSeriesResource
 import fs2.Stream
@@ -96,14 +96,14 @@ object TimeSeriesRelation
     with InsertSchema
     with NamedRelation
     with DeleteWithIdSchema
-    with UpdateSchemaFromUpsertSchema {
+    with UpdateSchema {
   override val name: String = "timeseries"
   import cognite.spark.compiletime.macros.StructTypeEncoderMacro._
 
   override val upsertSchema: StructType = structType[TimeSeriesUpsertSchema]()
   override val insertSchema: StructType = structType[TimeSeriesInsertSchema]()
   override val readSchema: StructType = structType[TimeSeriesReadSchema]()
-
+  override val updateSchema: StructType = upsertSchema
 }
 
 final case class TimeSeriesUpsertSchema(

src/test/scala/cognite/spark/v1/FileContentRelationTest.scala

@@ -129,19 +129,6 @@ class FileContentRelationTest  extends FlatSpec with Matchers with SparkTest wit
       )
   }
 
-  it should "fail with a sensible error if the mimetype is wrong" in {
-    val exception = sparkIntercept {
-      val sourceDf: DataFrame = dataFrameReaderUsingOidc
-        .useOIDCWrite
-        .option("type", "filecontent")
-        .option("externalId", fileWithWrongMimeTypeExternalId)
-        .load()
-      sourceDf.createOrReplaceTempView("fileContent")
-      spark.sqlContext.sql(s"select * from filecontent").collect()
-    }
-    assert(exception.getMessage.contains("Wrong mimetype. Expects application/jsonlines"))
-  }
-
   it should "infer the schema" in {
     val sourceDf: DataFrame = dataFrameReaderUsingOidc
       .useOIDCWrite

src/test/scala/cognite/spark/v1/SttpClientBackendFactoryTest.scala

@@ -16,5 +16,4 @@ class SttpClientBackendFactoryTest extends FlatSpec with Matchers {
     asyncHttpClient.close()
     asyncHttpClient.isClosed shouldBe true
   }
-
 }