3.4+

.github/workflows/build.yml

@@ -11,25 +11,8 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        scala: [ "2.12.17", "2.13.10" ]
-        spark: [ "3.3.2", "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
-        exclude:
-          - scala: "2.13.10"
-            spark: "3.1.3"
-          - scala: "2.13.10"
-            spark: "3.1.2"
-          - scala: "2.13.10"
-            spark: "3.1.1"
-          - scala: "2.13.10"
-            spark: "3.1.0"
-          - scala: "2.13.10"
-            spark: "3.0.3"
-          - scala: "2.13.10"
-            spark: "3.0.2"
-          - scala: "2.13.10"
-            spark: "3.0.1"
-          - scala: "2.13.10"
-            spark: "3.0.0"
+        scala: [ "2.12.19", "2.13.13" ]
+        spark: [ "3.4.2", "3.5.1" ]
     runs-on: ubuntu-latest
 
     steps:
@@ -49,7 +32,7 @@ jobs:
             ~/.gradle/caches
             ~/.gradle/wrapper
             ~/.gradle/jdks
-          key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+          key: "${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}"
           restore-keys: |
             ${{ runner.os }}-gradle-
 

.github/workflows/generate_docs.yml

@@ -26,7 +26,7 @@ jobs:
             ~/.gradle/caches
             ~/.gradle/wrapper
             ~/.gradle/jdks
-          key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+          key: "${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}"
           restore-keys: |
             ${{ runner.os }}-gradle-
 

.github/workflows/publish_dev_version.yml

@@ -9,25 +9,8 @@ jobs:
   build-and-deploy:
     strategy:
       matrix:
-        scala: [ "2.12.17", "2.13.10" ]
-        spark: [ "3.3.2", "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
-        exclude:
-          - scala: "2.13.10"
-            spark: "3.1.3"
-          - scala: "2.13.10"
-            spark: "3.1.2"
-          - scala: "2.13.10"
-            spark: "3.1.1"
-          - scala: "2.13.10"
-            spark: "3.1.0"
-          - scala: "2.13.10"
-            spark: "3.0.3"
-          - scala: "2.13.10"
-            spark: "3.0.2"
-          - scala: "2.13.10"
-            spark: "3.0.1"
-          - scala: "2.13.10"
-            spark: "3.0.0"
+        scala: [ "2.12.19", "2.13.13" ]
+        spark: [ "3.4.2", "3.5.1" ]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -50,7 +33,7 @@ jobs:
             ~/.gradle/caches
             ~/.gradle/wrapper
             ~/.gradle/jdks
-          key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+          key: "${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}"
           restore-keys: |
             ${{ runner.os }}-gradle-
 
@@ -72,7 +55,7 @@ jobs:
           ./gradlew 
           -Pspark=${{ matrix.spark }} 
           -Pscala=${{ matrix.scala }} 
-          -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.10' && matrix.spark == '3.2.0') }} 
+          -PskipScalaOnlyDependent=${{ matrix.spark != matrix.spark[0] }}
           clean 
           publishMavenPublicationToGitHubPackagesRepository 
           --scan

.github/workflows/publish_release_version.yml

@@ -8,25 +8,8 @@ jobs:
   build-and-deploy-mvn-central:
     strategy:
       matrix:
-        scala: [ "2.12.17", "2.13.10" ]
-        spark: [ "3.3.2", "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
-        exclude:
-          - scala: "2.13.10"
-            spark: "3.1.3"
-          - scala: "2.13.10"
-            spark: "3.1.2"
-          - scala: "2.13.10"
-            spark: "3.1.1"
-          - scala: "2.13.10"
-            spark: "3.1.0"
-          - scala: "2.13.10"
-            spark: "3.0.3"
-          - scala: "2.13.10"
-            spark: "3.0.2"
-          - scala: "2.13.10"
-            spark: "3.0.1"
-          - scala: "2.13.10"
-            spark: "3.0.0"
+        scala: [ "2.12.19", "2.13.13" ]
+        spark: [ "3.4.2", "3.5.1" ]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -49,7 +32,7 @@ jobs:
             ~/.gradle/caches
             ~/.gradle/wrapper
             ~/.gradle/jdks
-          key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+          key: "${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}"
           restore-keys: |
             ${{ runner.os }}-gradle-
 
@@ -74,7 +57,7 @@ jobs:
           ./gradlew 
           -Pspark=${{ matrix.spark }} 
           -Pscala=${{ matrix.scala }} 
-          -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.10' && matrix.spark == '3.2.0') }} 
+          -PskipScalaOnlyDependent=${{ matrix.spark != matrix.spark[0] }} 
           clean 
           publishMavenPublicationToMavenCentralRepository
           --scan

build.gradle.kts

@@ -1,21 +1,31 @@
 @file:Suppress("UnstableApiUsage")
 
+import Projects.compilerPlugin
+import Projects.gradlePlugin
+import com.github.gmazzo.buildconfig.BuildConfigExtension
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+
 buildscript {
     repositories {
         mavenCentral()
     }
     dependencies {
         classpath(jcp)
         classpath(mavenPublish)
+
+        // Allows the project to use the gradle plugin without mavenLocal
+        // Kept up-to-date by :gradle-plugin:updateBootstrapVersion
+        classpath(files("${project.rootDir.absolutePath}/gradle/bootstraps/gradle-plugin.jar"))
     }
 }
 
-
 plugins {
     mavenPublish version Versions.mavenPublish
     dokka version Versions.dokka
     idea
     kotlin version Versions.kotlin apply false
+    buildconfig version Versions.buildconfig apply false
 }
 
 group = Versions.groupID
@@ -114,4 +124,55 @@ allprojects {
             }
         }
     }
+}
+
+subprojects {
+    afterEvaluate {
+        // Adding the bootstraps directory to the repositories of the subprojects, so that
+        // the bootstrap version of compiler-plugin.jar can be found and used by the gradle-plugin
+        // without mavenLocal
+        if (plugins.hasPlugin("org.jetbrains.kotlinx.spark.api")) {
+            repositories.flatDir {
+                dirs("${project.rootDir.absolutePath}/gradle/bootstraps")
+            }
+            tasks.withType<KotlinCompile> {
+                dependsOn(":compiler-plugin:updateBootstrapVersion")
+                dependsOn(":gradle-plugin:updateBootstrapVersion")
+            }
+        }
+
+        repositories.flatDir {
+            dirs("${project.rootDir.absolutePath}/gradle/bootstraps")
+        }
+        extensions.findByType<BuildConfigExtension>()?.apply {
+            val projectVersion = Versions.project
+            val groupId = Versions.groupID
+
+            val compilerPluginArtifactId = compilerPlugin.name
+            val gradlePluginArtifactId = gradlePlugin.name
+
+            val compilerPluginId = "$groupId.api"
+
+            val defaultSparkifyFqName = "$groupId.api.plugin.annotations.Sparkify"
+            val defaultColumnNameFqName = "$groupId.api.plugin.annotations.ColumnName"
+
+            val projectRoot = project.rootDir.absolutePath
+
+            packageName("$groupId.api")
+            className("Artifacts")
+
+            buildConfigField("compilerPluginId", compilerPluginId)
+            buildConfigField("groupId", groupId)
+            buildConfigField("gradlePluginArtifactId", gradlePluginArtifactId)
+            buildConfigField("projectVersion", projectVersion)
+            buildConfigField("compilerPluginArtifactId", compilerPluginArtifactId)
+
+            buildConfigField("defaultSparkifyFqName", defaultSparkifyFqName)
+            buildConfigField("defaultColumnNameFqName", defaultColumnNameFqName)
+            buildConfigField("projectRoot", projectRoot)
+
+            buildConfigField("scalaVersion", Versions.scala)
+            buildConfigField("sparkVersion", Versions.spark)
+        }
+    }
 }

buildSrc/build.gradle.kts

@@ -1,10 +1,8 @@
-import org.gradle.kotlin.dsl.`kotlin-dsl`
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
-
 plugins {
     `kotlin-dsl`
 }
 
 repositories {
     mavenCentral()
+    maven("https://maven.pkg.jetbrains.space/kotlin/p/kotlin/bootstrap")
 }

buildSrc/src/main/kotlin/Dependencies.kt

@@ -1,15 +1,26 @@
-object Dependencies {
+object Dependencies : Dsl<Dependencies> {
     inline val kotlinStdLib get() = "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${Versions.kotlin}"
     inline val reflect get() = "org.jetbrains.kotlin:kotlin-reflect:${Versions.kotlin}"
     inline val scalaLibrary get() = "org.scala-lang:scala-library:${Versions.scala}"
     inline val kotlinxHtml get() = "org.jetbrains.kotlinx:kotlinx-html-jvm:${Versions.kotlinxHtml}"
     inline val sparkSql get() = "org.apache.spark:spark-sql_${Versions.scalaCompat}:${Versions.spark}"
+    inline val sparkSqlApi get() = "org.apache.spark:spark-sql-api_${Versions.scalaCompat}:${Versions.spark}"
+    inline val sparkConnectClient get() = "org.apache.spark:spark-connect-client-jvm_${Versions.scalaCompat}:${Versions.spark}"
     inline val sparkMl get() = "org.apache.spark:spark-mllib_${Versions.scalaCompat}:${Versions.spark}"
     inline val sparkStreaming get() = "org.apache.spark:spark-streaming_${Versions.scalaCompat}:${Versions.spark}"
     inline val hadoopClient get() = "org.apache.hadoop:hadoop-client:${Versions.hadoop}"
     inline val sparkRepl get() = "org.apache.spark:spark-repl_${Versions.scalaCompat}:${Versions.spark}"
     inline val jupyter get() = "org.jetbrains.kotlinx:kotlin-jupyter-api:${Versions.jupyter}"
-    inline val junit get() = "org.junit.jupiter:junit-jupiter-engine:5.8.1"
+    inline val junitJupiterEngine get() = "org.junit.jupiter:junit-jupiter-engine:${Versions.junitJupiterEngine}"
+    // must be platform()
+    inline val junitBom get() = "org.junit:junit-bom:${Versions.junitJupiterEngine}"
+    inline val junitJupiter get() = "org.junit.jupiter:junit-jupiter"
+    inline val junitPlatformCommons get() = "org.junit.platform:junit-platform-commons"
+    inline val junitPlatformLauncher get() = "org.junit.platform:junit-platform-launcher"
+    inline val junitPlatformRunner get() = "org.junit.platform:junit-platform-runner"
+    inline val junitPlatformSuiteApi get() = "org.junit.platform:junit-platform-suite-api"
+
+    inline val junit get() = "junit:junit:${Versions.junit}"
     inline val sparkStreamingKafka get() = "org.apache.spark:spark-streaming-kafka-0-10_${Versions.scalaCompat}:${Versions.spark}"
     inline val kotest get() = "io.kotest:kotest-runner-junit5:${Versions.kotest}"
     inline val kotestTestcontainers get() = "io.kotest.extensions:kotest-extensions-testcontainers:${Versions.kotestTestContainers}"
@@ -21,6 +32,12 @@ object Dependencies {
     inline val kotlinScriptingCommon get() = "org.jetbrains.kotlin:kotlin-scripting-common"
     inline val kotlinScriptingJvm get() = "org.jetbrains.kotlin:kotlin-scripting-jvm"
     inline val jacksonDatabind get() = "com.fasterxml.jackson.core:jackson-databind:${Versions.jacksonDatabind}"
+    inline val kotlinDateTime get() = "org.jetbrains.kotlinx:kotlinx-datetime:${Versions.kotlinxDateTime}"
+    inline val kotlinCompiler get() = "org.jetbrains.kotlin:kotlin-compiler:${Versions.kotlin}"
+    inline val kotlinScriptRuntime get() = "org.jetbrains.kotlin:kotlin-script-runtime:${Versions.kotlin}"
+    inline val kotlinAnnotationsJvm get() = "org.jetbrains.kotlin:kotlin-annotations-jvm:${Versions.kotlin}"
+    inline val kotlinCompilerInternalTestFramework get() = "org.jetbrains.kotlin:kotlin-compiler-internal-test-framework:${Versions.kotlin}"
+    inline val kotlinGradlePlugin get() = "org.jetbrains.kotlin:kotlin-gradle-plugin"
 }
 
 

buildSrc/src/main/kotlin/Helpers.kt

@@ -2,6 +2,11 @@ import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.dsl.DependencyHandler
 
+interface Dsl<T> {
+    @Suppress("UNCHECKED_CAST")
+    operator fun invoke(block: T.() -> Unit) = block(this as T)
+}
+
 fun DependencyHandler.testApi(vararg dependencyNotations: Any): List<Dependency?> =
     dependencyNotations.map {
         add("testApi", it)
@@ -18,6 +23,11 @@ fun DependencyHandler.testImplementation(vararg dependencyNotations: Any): List<
         add("testImplementation", it)
     }
 
+fun DependencyHandler.testRuntimeOnly(vararg dependencyNotations: Any): List<Dependency?> =
+    dependencyNotations.map {
+        add("testRuntimeOnly", it)
+    }
+
 fun DependencyHandler.implementation(vararg dependencyNotations: Any): List<Dependency?> =
     dependencyNotations.map {
         add("implementation", it)
@@ -28,10 +38,15 @@ fun DependencyHandler.runtimeOnly(vararg dependencyNotations: Any): List<Depende
         add("runtimeOnly", it)
     }
 
+fun DependencyHandler.compileOnly(vararg dependencyNotations: Any): List<Dependency?> =
+    dependencyNotations.map {
+        add("compileOnly", it)
+    }
+
 fun DependencyHandler.project(
     path: String,
     configuration: String? = null
 ): ProjectDependency = project(
     if (configuration != null) mapOf("path" to path, "configuration" to configuration)
     else mapOf("path" to path)
-) as ProjectDependency
+) as ProjectDependency

buildSrc/src/main/kotlin/Plugins.kt

@@ -2,6 +2,8 @@ import org.gradle.api.Project
 import org.gradle.kotlin.dsl.*
 import org.gradle.plugin.use.PluginDependenciesSpec
 
+inline val PluginDependenciesSpec.kotlinSparkApi
+    get() = id("org.jetbrains.kotlinx.spark.api")
 
 inline val PluginDependenciesSpec.kotlin
     get() = kotlin("jvm")
@@ -33,3 +35,11 @@ inline val Project.mavenPublishBase
 inline val PluginDependenciesSpec.jupyter
     get() = kotlin("jupyter.api") version Versions.jupyter
 
+inline val PluginDependenciesSpec.buildconfig
+    get() = id("com.github.gmazzo.buildconfig")
+
+inline val PluginDependenciesSpec.gradlePublishPlugin
+    get() = id("com.gradle.plugin-publish") version Versions.gradlePublishPlugin
+
+inline val PluginDependenciesSpec.shadow
+    get() = id("com.github.johnrengelman.shadow") version Versions.shadow

buildSrc/src/main/kotlin/Projects.kt

@@ -1,10 +1,8 @@
 @file:Suppress("NOTHING_TO_INLINE")
 
 import org.gradle.api.Project
-import org.gradle.api.artifacts.dsl.DependencyHandler
-import org.gradle.kotlin.dsl.support.delegates.ProjectDelegate
 
-object Projects {
+object Projects : Dsl<Projects> {
 
     inline fun Project.searchProject(name: String): Project =
         rootProject
@@ -17,8 +15,8 @@ object Projects {
     inline val Project.kotlinSparkApi
         get() = searchProject("kotlin-spark-api")
 
-    inline val Project.core
-        get() = searchProject("core")
+    inline val Project.scalaHelpers
+        get() = searchProject("scala-helpers")
 
     inline val Project.examples
         get() = searchProject("examples")
@@ -28,4 +26,10 @@ object Projects {
 
     inline val Project.scalaTuplesInKotlin
         get() = searchProject("scala-tuples-in-kotlin")
+
+    inline val Project.compilerPlugin
+        get() = searchProject("compiler-plugin")
+
+    inline val Project.gradlePlugin
+        get() = searchProject("gradle-plugin")
 }