Merge branch 'master' into renovate/lock-file-maintenance

README.md

@@ -1474,6 +1474,8 @@ Optionally, you can have Spark infer the DataFrame schema with the following opt
 
 - `rawEnsureParent`: When set to true, the parent database and table will be creates if it does not exists already.
 
+- `filterNullFieldsOnNonSchemaRawQueries`: Set this to `"true"`to enable experimental support for filtering empty columns server side in the Raw API, without impacting the inferred schema. Aimed to become enabled by default in the future once it has been fully tested.
+
 ```scala
 val df = spark.read.format("cognite.spark.v1")
   .option("type", "raw")

build.sbt

@@ -11,7 +11,7 @@ val circeVersion = "0.14.9"
 val sttpVersion = "3.5.2"
 val natchezVersion = "0.3.1"
 val Specs2Version = "4.20.3"
-val cogniteSdkVersion = "2.28.844"
+val cogniteSdkVersion = "2.31.861"
 
 val prometheusVersion = "0.16.0"
 val log4sVersion = "1.10.0"
@@ -41,7 +41,7 @@ lazy val commonSettings = Seq(
   organization := "com.cognite.spark.datasource",
   organizationName := "Cognite",
   organizationHomepage := Some(url("https://cognite.com")),
-  version := "3.20." + patchVersion,
+  version := "3.21." + patchVersion,
   isSnapshot := patchVersion.endsWith("-SNAPSHOT"),
   crossScalaVersions := supportedScalaVersions,
   semanticdbEnabled := true,
@@ -160,6 +160,7 @@ lazy val library = (project in file("."))
         exclude("org.scala-lang.modules", "scala-collection-compat_2.12")
         exclude("org.scala-lang.modules", "scala-collection-compat_2.13"),
       "org.specs2" %% "specs2-core" % Specs2Version % Test,
+      "com.softwaremill.sttp.client3" %% "async-http-client-backend-fs2" % sttpVersion,
       "com.softwaremill.sttp.client3" %% "async-http-client-backend-cats" % sttpVersion
         // Netty is included in Spark as jars/netty-all-4.<minor>.<patch>.Final.jar
         exclude("io.netty", "netty-buffer")

build.scala-2.12.sbt.lock

@@ -1,6 +1,6 @@
 {
   "lockVersion" : 1,
-  "timestamp" : "2024-08-20T11:20:27.551661Z",
+  "timestamp" : "2024-11-13T11:30:40.138510863Z",
   "configurations" : [
     "compile",
     "optional",
@@ -13,11 +13,11 @@
     {
       "org" : "co.fs2",
       "name" : "fs2-core_2.12",
-      "version" : "3.10.2",
+      "version" : "3.11.0",
       "artifacts" : [
         {
           "name" : "fs2-core_2.12.jar",
-          "hash" : "sha1:413c7d123f943eea68f6c77907318e58e0410ad4"
+          "hash" : "sha1:3651a652ea778dea12d9ec8083896f05ba489439"
         }
       ],
       "configurations" : [
@@ -30,11 +30,11 @@
     {
       "org" : "co.fs2",
       "name" : "fs2-io_2.12",
-      "version" : "3.10.2",
+      "version" : "3.11.0",
       "artifacts" : [
         {
           "name" : "fs2-io_2.12.jar",
-          "hash" : "sha1:89cefd0c05480d603e26d4b92a3d8284b8ec7b4a"
+          "hash" : "sha1:eb83f8a4387e935926d00d81999adbae49756a30"
         }
       ],
       "configurations" : [
@@ -44,6 +44,22 @@
         "test"
       ]
     },
+    {
+      "org" : "co.fs2",
+      "name" : "fs2-reactive-streams_2.12",
+      "version" : "3.2.7",
+      "artifacts" : [
+        {
+          "name" : "fs2-reactive-streams_2.12.jar",
+          "hash" : "sha1:4b959071b733d9759dcb324dd15216003e511f41"
+        }
+      ],
+      "configurations" : [
+        "compile",
+        "runtime",
+        "test"
+      ]
+    },
     {
       "org" : "com.chuusai",
       "name" : "shapeless_2.12",
@@ -78,11 +94,11 @@
     {
       "org" : "com.cognite",
       "name" : "cognite-sdk-scala_2.12",
-      "version" : "2.28.844",
+      "version" : "2.31.861",
       "artifacts" : [
         {
           "name" : "cognite-sdk-scala_2.12.jar",
-          "hash" : "sha1:a020427c9c31f27e6ec0028f0f3e12f4b8e6fc8f"
+          "hash" : "sha1:ae52606e613a31e1979830a6036f265328033908"
         }
       ],
       "configurations" : [
@@ -95,11 +111,11 @@
     {
       "org" : "com.comcast",
       "name" : "ip4s-core_2.12",
-      "version" : "3.5.0",
+      "version" : "3.6.0",
       "artifacts" : [
         {
           "name" : "ip4s-core_2.12.jar",
-          "hash" : "sha1:19d582e20c3cb27c1481b7e43baea0fa90aa7cd6"
+          "hash" : "sha1:d5b15a847dbd0ca35a281d140ef4d26b00d33b60"
         }
       ],
       "configurations" : [
@@ -324,6 +340,22 @@
         "test"
       ]
     },
+    {
+      "org" : "com.softwaremill.sttp.client3",
+      "name" : "async-http-client-backend-fs2_2.12",
+      "version" : "3.5.2",
+      "artifacts" : [
+        {
+          "name" : "async-http-client-backend-fs2_2.12.jar",
+          "hash" : "sha1:fadae6ca5d30cd1d2eb90ad18c679969f027a94e"
+        }
+      ],
+      "configurations" : [
+        "compile",
+        "runtime",
+        "test"
+      ]
+    },
     {
       "org" : "com.softwaremill.sttp.client3",
       "name" : "async-http-client-backend_2.12",
@@ -390,6 +422,22 @@
         "test"
       ]
     },
+    {
+      "org" : "com.softwaremill.sttp.client3",
+      "name" : "fs2_2.12",
+      "version" : "3.5.2",
+      "artifacts" : [
+        {
+          "name" : "fs2_2.12.jar",
+          "hash" : "sha1:8dde4d364d013efd245e5d4b849ff3b1e202d5f2"
+        }
+      ],
+      "configurations" : [
+        "compile",
+        "runtime",
+        "test"
+      ]
+    },
     {
       "org" : "com.softwaremill.sttp.client3",
       "name" : "json-common_2.12",
@@ -441,6 +489,22 @@
         "test"
       ]
     },
+    {
+      "org" : "com.softwaremill.sttp.shared",
+      "name" : "fs2_2.12",
+      "version" : "1.3.4",
+      "artifacts" : [
+        {
+          "name" : "fs2_2.12.jar",
+          "hash" : "sha1:a53a923369b494da9283bd0bb9c34639f9d63f8a"
+        }
+      ],
+      "configurations" : [
+        "compile",
+        "runtime",
+        "test"
+      ]
+    },
     {
       "org" : "com.softwaremill.sttp.shared",
       "name" : "ws_2.12",
@@ -577,11 +641,11 @@
     {
       "org" : "commons-io",
       "name" : "commons-io",
-      "version" : "2.16.1",
+      "version" : "2.17.0",
       "artifacts" : [
         {
           "name" : "commons-io.jar",
-          "hash" : "sha1:377d592e740dc77124e0901291dbfaa6810a200e"
+          "hash" : "sha1:ddcc8433eb019fb48fe25207c0278143f3e1d7e2"
         }
       ],
       "configurations" : [
@@ -818,6 +882,22 @@
         "provided"
       ]
     },
+    {
+      "org" : "io.netty",
+      "name" : "netty-buffer",
+      "version" : "4.1.60.Final",
+      "artifacts" : [
+        {
+          "name" : "netty-buffer.jar",
+          "hash" : "sha1:9d213d090deeca2541ad6827eb3345bcd6e1e701"
+        }
+      ],
+      "configurations" : [
+        "compile",
+        "runtime",
+        "test"
+      ]
+    },
     {
       "org" : "io.netty",
       "name" : "netty-buffer",
@@ -924,6 +1004,22 @@
         "provided"
       ]
     },
+    {
+      "org" : "io.netty",
+      "name" : "netty-handler",
+      "version" : "4.1.60.Final",
+      "artifacts" : [
+        {
+          "name" : "netty-handler.jar",
+          "hash" : "sha1:14e28bab0173be10c9631a85069636a0d0221dfe"
+        }
+      ],
+      "configurations" : [
+        "compile",
+        "runtime",
+        "test"
+      ]
+    },
     {
       "org" : "io.netty",
       "name" : "netty-handler",
@@ -2817,11 +2913,11 @@
     {
       "org" : "org.typelevel",
       "name" : "cats-core_2.12",
-      "version" : "2.10.0",
+      "version" : "2.11.0",
       "artifacts" : [
         {
           "name" : "cats-core_2.12.jar",
-          "hash" : "sha1:100bc0d103c5bd9ddcd4e2b5c24e88483e2a77d5"
+          "hash" : "sha1:44f89a95aed47e43f85896b1243b7dc6a7e1feae"
         }
       ],
       "configurations" : [
@@ -2885,11 +2981,11 @@
     {
       "org" : "org.typelevel",
       "name" : "cats-kernel_2.12",
-      "version" : "2.10.0",
+      "version" : "2.11.0",
       "artifacts" : [
         {
           "name" : "cats-kernel_2.12.jar",
-          "hash" : "sha1:94b372f873e763186c3cccc9e76492553d01350e"
+          "hash" : "sha1:dd67f32ab72c3aa2045432992cdf3f0a42484645"
         }
       ],
       "configurations" : [