From beb004025d78db6778d0408277cc71939e92c711 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 11:54:54 +0800 Subject: [PATCH 01/13] Add build for custom dialect --- .github/workflows/odps.yaml | 34 +++++++++++++++++++ caraml-store-spark/build.gradle | 10 ++++++ .../dev/caraml/spark/odps/CustomDialect.scala | 12 +++++++ 3 files changed, 56 insertions(+) create mode 100644 .github/workflows/odps.yaml create mode 100644 caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml new file mode 100644 index 0000000..1d54ce7 --- /dev/null +++ b/.github/workflows/odps.yaml @@ -0,0 +1,34 @@ +name: Publish CustomDialect JAR + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: "17" + distribution: "temurin" + + - name: Build customDialectJar + uses: gradle/gradle-build-action@v2 + with: + arguments: customDialectJar + + - name: Publish customDialectJar + uses: actions/upload-artifact@v2 + with: + name: custom-dialect + path: build/libs/custom-dialect.jar diff --git a/caraml-store-spark/build.gradle b/caraml-store-spark/build.gradle index c3616aa..af4b4a1 100644 --- a/caraml-store-spark/build.gradle +++ b/caraml-store-spark/build.gradle @@ -148,3 +148,13 @@ sourceSets { } } } + + +// Define a new task to package CustomDialect as an individual JAR +task customDialectJar(type: Jar) { + archiveBaseName.set('custom-dialect') + from sourceSets.main.output + include 'dev/caraml/spark/odps/CustomDialect.class' + destinationDirectory.set(file("$buildDir/libs")) +} +customDialectJar.dependsOn classes \ No newline at end of file diff --git a/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala b/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala new file mode 100644 index 0000000..aa351cd --- /dev/null +++ b/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala @@ -0,0 +1,12 @@ +package dev.caraml.spark.odps +import org.apache.spark.sql.jdbc.JdbcDialect + +class CustomDialect extends JdbcDialect { + override def canHandle(url: String): Boolean = { + url.startsWith("jdbc:odps") + } + + override def quoteIdentifier(colName: String): String = { + s"`$colName`" + } +} \ No newline at end of file From 2220ab055e0b30b8202ad926bce505d66d119a8a Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 12:19:32 +0800 Subject: [PATCH 02/13] Use upload artifact v4 --- .github/workflows/odps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml index 1d54ce7..716488e 100644 --- a/.github/workflows/odps.yaml +++ b/.github/workflows/odps.yaml @@ -28,7 +28,7 @@ jobs: arguments: customDialectJar - name: Publish customDialectJar - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: custom-dialect path: build/libs/custom-dialect.jar From e132353af5201906820840bbeca6b79fd50a625b Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 12:43:48 +0800 Subject: [PATCH 03/13] Fix build path --- .github/workflows/odps.yaml | 2 +- caraml-store-spark/build.gradle | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml index 716488e..afb58c7 100644 --- a/.github/workflows/odps.yaml +++ b/.github/workflows/odps.yaml @@ -31,4 +31,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: custom-dialect - path: build/libs/custom-dialect.jar + path: caraml-store-spark/build/libs/custom-dialect.jar diff --git a/caraml-store-spark/build.gradle b/caraml-store-spark/build.gradle index af4b4a1..a6955f1 100644 --- a/caraml-store-spark/build.gradle +++ b/caraml-store-spark/build.gradle @@ -153,8 +153,13 @@ sourceSets { // Define a new task to package CustomDialect as an individual JAR task customDialectJar(type: Jar) { archiveBaseName.set('custom-dialect') + archiveVersion.set('') + archiveFileName.set('custom-dialect.jar') from sourceSets.main.output include 'dev/caraml/spark/odps/CustomDialect.class' destinationDirectory.set(file("$buildDir/libs")) + doLast { + println "Path: ${destinationDirectory.get().asFile}/custom-dialect.jar" + } } customDialectJar.dependsOn classes \ No newline at end of file From 54a61c6f33686dde5f7cd79d2ec49ebccee71a09 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 17:00:05 +0800 Subject: [PATCH 04/13] Add step to publish jar as package in caraml-store --- .github/workflows/odps.yaml | 7 +++++++ caraml-store-spark/build.gradle | 26 +++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml index afb58c7..5e4cc5d 100644 --- a/.github/workflows/odps.yaml +++ b/.github/workflows/odps.yaml @@ -32,3 +32,10 @@ jobs: with: name: custom-dialect path: caraml-store-spark/build/libs/custom-dialect.jar + + - name: Publish customDialectJar to GitHub Packages + uses: gradle/gradle-build-action@v2 + with: + arguments: caraml-store-spark:customDialectJar + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/caraml-store-spark/build.gradle b/caraml-store-spark/build.gradle index a6955f1..1bda758 100644 --- a/caraml-store-spark/build.gradle +++ b/caraml-store-spark/build.gradle @@ -6,6 +6,7 @@ plugins { id 'com.github.maiflai.scalatest' version "0.32" id 'com.google.protobuf' id 'idea' + id 'maven-publish' } ext { @@ -162,4 +163,27 @@ task customDialectJar(type: Jar) { println "Path: ${destinationDirectory.get().asFile}/custom-dialect.jar" } } -customDialectJar.dependsOn classes \ No newline at end of file +customDialectJar.dependsOn classes + + +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + artifactId = 'custom-dialect' + artifact("$buildDir/libs/custom-dialect.jar") { + builtBy customDialectJar + } + } + } + repositories { + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/caraml-dev/caraml-store") + credentials { + username = System.getenv("GITHUB_ACTOR") + password = System.getenv("GITHUB_TOKEN") + } + } + } +} \ No newline at end of file From 10357eaaadf5c7555fc26c681628c9a6b9dab3d0 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 17:07:53 +0800 Subject: [PATCH 05/13] Fix odps.yaml workflow --- .github/workflows/odps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml index 5e4cc5d..9cbe8fb 100644 --- a/.github/workflows/odps.yaml +++ b/.github/workflows/odps.yaml @@ -36,6 +36,6 @@ jobs: - name: Publish customDialectJar to GitHub Packages uses: gradle/gradle-build-action@v2 with: - arguments: caraml-store-spark:customDialectJar + arguments: caraml-store-spark:publish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From ddb576b7ae1049998a72232665236ac282ac5781 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 17:33:47 +0800 Subject: [PATCH 06/13] Update build.gradle --- caraml-store-spark/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caraml-store-spark/build.gradle b/caraml-store-spark/build.gradle index 1bda758..c4e5728 100644 --- a/caraml-store-spark/build.gradle +++ b/caraml-store-spark/build.gradle @@ -168,7 +168,7 @@ customDialectJar.dependsOn classes publishing { publications { - mavenJava(MavenPublication) { + customDialect(MavenPublication) { from components.java artifactId = 'custom-dialect' artifact("$buildDir/libs/custom-dialect.jar") { From 705e6872a8053d36f5fcffcce48a6b5de4bc3709 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 18:37:38 +0800 Subject: [PATCH 07/13] Add getJDBCType mapping --- .../dev/caraml/spark/odps/CustomDialect.scala | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala b/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala index aa351cd..e07f137 100644 --- a/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala +++ b/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala @@ -1,5 +1,9 @@ package dev.caraml.spark.odps import org.apache.spark.sql.jdbc.JdbcDialect +import org.apache.spark.sql.jdbc.JdbcType +import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.types._ +import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ajax.JSON class CustomDialect extends JdbcDialect { override def canHandle(url: String): Boolean = { @@ -9,4 +13,32 @@ class CustomDialect extends JdbcDialect { override def quoteIdentifier(colName: String): String = { s"`$colName`" } -} \ No newline at end of file + + override def getJDBCType(dt: DataType): Option[JdbcType] = { + dt match { + case IntegerType => Option(JdbcType("INTEGER", java.sql.Types.INTEGER)) + case LongType => Option(JdbcType("BIGINT", java.sql.Types.BIGINT)) + case DoubleType => + Option(JdbcType("DOUBLE", java.sql.Types.DOUBLE)) + case FloatType => Option(JdbcType("FLOAT", java.sql.Types.FLOAT)) + case ShortType => Option(JdbcType("SMALLINT", java.sql.Types.SMALLINT)) + case ByteType => Option(JdbcType("TINYINT", java.sql.Types.TINYINT)) + case BooleanType => Option(JdbcType("BOOLEAN", java.sql.Types.BOOLEAN)) + case StringType => Option(JdbcType("STRING", java.sql.Types.CLOB)) + case BinaryType => Option(JdbcType("BINARY", java.sql.Types.BINARY)) + case TimestampType => + Option(JdbcType("TIMESTAMP", java.sql.Types.TIMESTAMP)) + case DateType => Option(JdbcType("DATE", java.sql.Types.DATE)) + case t: DecimalType => + Option( + JdbcType( + s"DECIMAL(${t.precision},${t.scale})", + java.sql.Types.DECIMAL + ) + ) + case VarcharType(length) => Option(JdbcType(s"VARCHAR($length)", java.sql.Types.VARCHAR)) + // NOTE: Update when necessary + case _ => None + } + } +} From aba024d9d511cc3523440e038d45d46b2b835591 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 21:52:34 +0800 Subject: [PATCH 08/13] Fix gradle --- caraml-store-spark/build.gradle | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/caraml-store-spark/build.gradle b/caraml-store-spark/build.gradle index c4e5728..90528b5 100644 --- a/caraml-store-spark/build.gradle +++ b/caraml-store-spark/build.gradle @@ -2,7 +2,7 @@ plugins { id 'caraml.scala' id 'application' id "com.github.johnrengelman.shadow" version "7.1.2" - id 'com.palantir.docker' version "0.34.0" + id 'com.palantir.docker' version "0.36.0" id 'com.github.maiflai.scalatest' version "0.32" id 'com.google.protobuf' id 'idea' @@ -169,7 +169,6 @@ customDialectJar.dependsOn classes publishing { publications { customDialect(MavenPublication) { - from components.java artifactId = 'custom-dialect' artifact("$buildDir/libs/custom-dialect.jar") { builtBy customDialectJar From a43476aa5383407b56d49d600218421ddf5f5e80 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 22:33:07 +0800 Subject: [PATCH 09/13] Change odps.yaml to trigger manually, draft release artifact --- .github/workflows/odps.yaml | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml index 9cbe8fb..f8b433e 100644 --- a/.github/workflows/odps.yaml +++ b/.github/workflows/odps.yaml @@ -1,13 +1,7 @@ name: Publish CustomDialect JAR on: - push: - branches: - - main - pull_request: - branches: - - main - + workflow_dispatch: # Allows manual triggering of the workflow jobs: build-and-publish: runs-on: ubuntu-latest @@ -33,9 +27,23 @@ jobs: name: custom-dialect path: caraml-store-spark/build/libs/custom-dialect.jar - - name: Publish customDialectJar to GitHub Packages - uses: gradle/gradle-build-action@v2 + - name: Create Draft Prerelease + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - arguments: caraml-store-spark:publish + tag_name: v${{ github.run_number }} + release_name: Release ${{ github.run_number }} + draft: true + prerelease: true + + - name: Upload customDialectJar to Release + uses: actions/upload-release-asset@v1 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: caraml-store-spark/build/libs/custom-dialect.jar + asset_name: custom-dialect.jar + asset_content_type: application/java-archive From 4d8333042a8b133a0a01c5c40c5044c11f90f2bc Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Mon, 3 Feb 2025 22:44:42 +0800 Subject: [PATCH 10/13] Add pr event --- .github/workflows/odps.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml index f8b433e..9ad0a9e 100644 --- a/.github/workflows/odps.yaml +++ b/.github/workflows/odps.yaml @@ -2,6 +2,7 @@ name: Publish CustomDialect JAR on: workflow_dispatch: # Allows manual triggering of the workflow + pull_request: jobs: build-and-publish: runs-on: ubuntu-latest From 0dc881f84bfa554bd85bb54e9706e3b632cb3d14 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Tue, 4 Feb 2025 12:13:43 +0800 Subject: [PATCH 11/13] Update odps workflow --- .github/workflows/odps.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml index 9ad0a9e..7ee4dea 100644 --- a/.github/workflows/odps.yaml +++ b/.github/workflows/odps.yaml @@ -2,7 +2,10 @@ name: Publish CustomDialect JAR on: workflow_dispatch: # Allows manual triggering of the workflow - pull_request: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+*" + jobs: build-and-publish: runs-on: ubuntu-latest @@ -34,8 +37,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: v${{ github.run_number }} - release_name: Release ${{ github.run_number }} + tag_name: ${{ github.ref_name }} + release_name: Release ${{ github.ref_name }} draft: true prerelease: true From 135f79fff1379cce7a7364c389fe65923e0c2cdf Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Tue, 4 Feb 2025 12:16:11 +0800 Subject: [PATCH 12/13] Trigger workflow on change sto spark/odps --- .github/workflows/odps.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/odps.yaml b/.github/workflows/odps.yaml index 7ee4dea..92836d0 100644 --- a/.github/workflows/odps.yaml +++ b/.github/workflows/odps.yaml @@ -4,7 +4,10 @@ on: workflow_dispatch: # Allows manual triggering of the workflow push: tags: - - "v[0-9]+.[0-9]+.[0-9]+*" + - "caraml-store-spark/odps/v[0-9]+.[0-9]+.[0-9]+*" + paths: + - ".github/workflows/odps.yaml" + - "caraml-store-spark/src/main/scala/dev/caraml/spark/odps/**" jobs: build-and-publish: From 9674a9702b6a629ac1d2eb152d4595020382ffd6 Mon Sep 17 00:00:00 2001 From: Shide Foo Date: Tue, 4 Feb 2025 13:14:54 +0800 Subject: [PATCH 13/13] Remove backtick for colname --- .../src/main/scala/dev/caraml/spark/odps/CustomDialect.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala b/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala index e07f137..8f571c0 100644 --- a/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala +++ b/caraml-store-spark/src/main/scala/dev/caraml/spark/odps/CustomDialect.scala @@ -11,7 +11,7 @@ class CustomDialect extends JdbcDialect { } override def quoteIdentifier(colName: String): String = { - s"`$colName`" + s"$colName" } override def getJDBCType(dt: DataType): Option[JdbcType] = {