[SPARK-50810][BUILD] Enable SBT CI for profiler module

### What changes were proposed in this pull request? Update GitHub workflow files and related scripts to enable SBT CI for the `profiler` module. ### Why are the changes needed? Currently, the `profiler` module is not covered in regular PR's SBT CI. ### Does this PR introduce _any_ user-facing change? No, dev only. ### How was this patch tested? https://github.com/pan3793/spark/actions/runs/12770426351/job/35595345586 ### Was this patch authored or co-authored using generative AI tooling? No. Closes apache#49480 from pan3793/SPARK-50810. Authored-by: Cheng Pan <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
G-Research · Jan 15, 2025 · 47d831e · 47d831e
1 parent 1517315
commit 47d831e
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 7 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -242,7 +242,7 @@ jobs:
           - >-
             api, catalyst, hive-thriftserver
           - >-
-            mllib-local, mllib, graphx
+            mllib-local, mllib, graphx, profiler
           - >-
             streaming, sql-kafka-0-10, streaming-kafka-0-10, streaming-kinesis-asl,
             kubernetes, hadoop-cloud, spark-ganglia-lgpl, protobuf, connect

diff --git a/connector/profiler/README.md b/connector/profiler/README.md
@@ -3,8 +3,15 @@
 ## Build
 
 To build
+
+```
+./build/mvn clean package -DskipTests -Pjvm-profiler -pl :spark-profiler_2.13 -am
+```
+
+or
+
 ```
-  ./build/mvn clean package -DskipTests -Pjvm-profiler
+./build/sbt -Pjvm-profiler clean "profiler/package"
 ```
 
 ## Executor Code Profiling

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
@@ -309,6 +309,15 @@ def __hash__(self):
     ],
 )
 
+profiler = Module(
+    name="profiler",
+    dependencies=[],
+    build_profile_flags=["-Pjvm-profiler"],
+    source_file_regexes=[
+        "connector/profiler",
+    ],
+)
+
 protobuf = Module(
     name="protobuf",
     dependencies=[sql],

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
@@ -65,10 +65,10 @@ object BuildCommons {
   ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ connectProjects
 
   val optionallyEnabledProjects@Seq(kubernetes, yarn,
-    sparkGangliaLgpl, streamingKinesisAsl,
+    sparkGangliaLgpl, streamingKinesisAsl, profiler,
     dockerIntegrationTests, hadoopCloud, kubernetesIntegrationTests) =
     Seq("kubernetes", "yarn",
-      "ganglia-lgpl", "streaming-kinesis-asl",
+      "ganglia-lgpl", "streaming-kinesis-asl", "profiler",
       "docker-integration-tests", "hadoop-cloud", "kubernetes-integration-tests").map(ProjectRef(buildLocation, _))
 
   val assemblyProjects@Seq(networkYarn, streamingKafka010Assembly, streamingKinesisAslAssembly) =
@@ -371,7 +371,7 @@ object SparkBuild extends PomBuild {
     Seq(
       spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn,
       unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon, connect, connectClient,
-      variant, connectShims
+      variant, connectShims, profiler
     ).contains(x)
   }
 
@@ -1469,11 +1469,11 @@ object SparkUnidoc extends SharedUnidocSettings {
     (ScalaUnidoc / unidoc / unidocProjectFilter) :=
       inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
         yarn, tags, streamingKafka010, sqlKafka010, connectCommon, connect, connectClient,
-        connectShims, protobuf),
+        connectShims, protobuf, profiler),
     (JavaUnidoc / unidoc / unidocProjectFilter) :=
       inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
         yarn, tags, streamingKafka010, sqlKafka010, connectCommon, connect, connectClient,
-        connectShims, protobuf),
+        connectShims, protobuf, profiler),
   )
 }