Automatically gather jstacks for Spark applications

.github/PULL_REQUEST_TEMPLATE

@@ -33,7 +33,8 @@ Please clarify why the changes are needed. For instance,
 
 ### Does this PR introduce _any_ user-facing change?
 <!--
-Note that it means *any* user-facing change including all aspects such as the documentation fix.
+Note that it means *any* user-facing change including all aspects such as new features, bug fixes, or other behavior changes. Documentation-only updates are not considered user-facing changes.
+
 If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
 If possible, please also clarify if this is a user-facing change compared to the released Spark versions or within the unreleased branches such as master.
 If no, write 'No'.

.github/labeler.yml

@@ -93,9 +93,9 @@ SQL:
   - changed-files:
     - all-globs-to-any-file: [
      '**/sql/**/*',
-     '!python/pyspark/sql/avro/**/*',
-     '!python/pyspark/sql/streaming/**/*',
-     '!python/pyspark/sql/tests/streaming/test_streaming*.py'
+     '!python/**/avro/**/*',
+     '!python/**/protobuf/**/*',
+     '!python/**/streaming/**/*'
     ]
     - any-glob-to-any-file: [
      'common/unsafe/**/*',
@@ -119,7 +119,7 @@ AVRO:
   - changed-files:
     - any-glob-to-any-file: [
      'connector/avro/**/*',
-     'python/pyspark/sql/avro/**/*'
+     'python/**/avro/**/*'
     ]
 
 DSTREAM:
@@ -152,18 +152,16 @@ ML:
 MLLIB:
   - changed-files:
     - any-glob-to-any-file: [
-     '**/spark/mllib/**/*',
-     'mllib-local/**/*',
-     'python/pyspark/mllib/**/*'
+     '**/mllib/**/*',
+     'mllib-local/**/*'
     ]
 
 STRUCTURED STREAMING:
   - changed-files:
     - any-glob-to-any-file: [
      '**/sql/**/streaming/**/*',
      'connector/kafka-0-10-sql/**/*',
-     'python/pyspark/sql/streaming/**/*',
-     'python/pyspark/sql/tests/streaming/test_streaming*.py',
+     'python/pyspark/sql/**/streaming/**/*',
      '**/*streaming.R'
     ]
 
@@ -225,14 +223,12 @@ CONNECT:
   - changed-files:
     - any-glob-to-any-file: [
      'sql/connect/**/*',
-     'connector/connect/**/*',
-     'python/pyspark/sql/**/connect/**/*',
-     'python/pyspark/ml/**/connect/**/*'
+     'python/**/connect/**/*'
     ]
 
 PROTOBUF:
   - changed-files:
     - any-glob-to-any-file: [
      'connector/protobuf/**/*',
-     'python/pyspark/sql/protobuf/**/*'
+     'python/**/protobuf/**/*'
     ]

.github/workflows/benchmark.yml

@@ -68,7 +68,7 @@ jobs:
   tpcds-1g-gen:
     name: "Generate an input dataset for TPCDSQueryBenchmark with SF=1"
     if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, '*')
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -105,7 +105,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: databricks/tpcds-kit
-          ref: 2a5078a782192ddb6efbcead8de9973d6ab4f069
+          ref: 1b7fb7529edae091684201fab142d956d6afd881
           path: ./tpcds-kit
       - name: Build tpcds-kit
         if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'

.github/workflows/build_branch35.yml

@@ -22,6 +22,7 @@ name: "Build (branch-3.5, Scala 2.13, Hadoop 3, JDK 8)"
 on:
   schedule:
     - cron: '0 11 * * *'
+  workflow_dispatch:
 
 jobs:
   run-build:
@@ -37,6 +38,7 @@ jobs:
       envs: >-
         {
           "SCALA_PROFILE": "scala2.13",
+          "PYSPARK_IMAGE_TO_TEST": "",
           "PYTHON_TO_TEST": "",
           "ORACLE_DOCKER_IMAGE_NAME": "gvenzl/oracle-xe:21.3.0"
         }

.github/workflows/build_branch35_python.yml

@@ -22,6 +22,7 @@ name: "Build / Python-only (branch-3.5)"
 on:
   schedule:
     - cron: '0 11 * * *'
+  workflow_dispatch:
 
 jobs:
   run-build:
@@ -36,6 +37,7 @@ jobs:
       hadoop: hadoop3
       envs: >-
         {
+          "PYSPARK_IMAGE_TO_TEST": "",
           "PYTHON_TO_TEST": ""
         }
       jobs: >-

.github/workflows/build_branch40.yml

@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Build (branch-4.0, Scala 2.13, Hadoop 3, JDK 17)"
+
+on:
+  schedule:
+    - cron: '0 12 * * *'
+  workflow_dispatch:
+
+jobs:
+  run-build:
+    permissions:
+      packages: write
+    name: Run
+    uses: ./.github/workflows/build_and_test.yml
+    if: github.repository == 'apache/spark'
+    with:
+      java: 17
+      branch: branch-4.0
+      hadoop: hadoop3
+      envs: >-
+        {
+          "SCALA_PROFILE": "scala2.13",
+          "PYSPARK_IMAGE_TO_TEST": "",
+          "PYTHON_TO_TEST": "",
+          "ORACLE_DOCKER_IMAGE_NAME": "gvenzl/oracle-free:23.6-slim"
+        }
+      jobs: >-
+        {
+          "build": "true",
+          "sparkr": "true",
+          "tpcds-1g": "true",
+          "docker-integration-tests": "true",
+          "k8s-integration-tests": "true",
+          "lint" : "true"
+        }

.github/workflows/build_branch40_java21.yml

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Build (branch-4.0, Scala 2.13, Hadoop 3, JDK 21)"
+
+on:
+  schedule:
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+
+jobs:
+  run-build:
+    permissions:
+      packages: write
+    name: Run
+    uses: ./.github/workflows/build_and_test.yml
+    if: github.repository == 'apache/spark'
+    with:
+      java: 21
+      branch: branch-4.0
+      hadoop: hadoop3
+      envs: >-
+        {
+          "PYSPARK_IMAGE_TO_TEST": "python-311",
+          "PYTHON_TO_TEST": "python3.11",
+          "SKIP_MIMA": "true",
+          "SKIP_UNIDOC": "true",
+          "DEDICATED_JVM_SBT_TESTS": "org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV1Suite,org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV2Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV1Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV2Suite"
+        }
+      jobs: >-
+        {
+          "build": "true",
+          "pyspark": "true",
+          "sparkr": "true",
+          "tpcds-1g": "true",
+          "docker-integration-tests": "true",
+          "yarn": "true",
+          "k8s-integration-tests": "true",
+          "buf": "true",
+          "ui": "true"
+        }

.github/workflows/build_branch40_maven.yml

@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Build / Maven (branch-4.0, Scala 2.13, Hadoop 3, JDK 17)"
+
+on:
+  schedule:
+    - cron: '0 14 * * *'
+  workflow_dispatch:
+
+jobs:
+  run-build:
+    permissions:
+      packages: write
+    name: Run
+    uses: ./.github/workflows/maven_test.yml
+    if: github.repository == 'apache/spark'
+    with:
+      branch: branch-4.0

.github/workflows/build_branch40_maven_java21.yml

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Build / Maven (branch-4.0, Scala 2.13, Hadoop 3, JDK 21)"
+
+on:
+  schedule:
+    - cron: '0 14 * * *'
+  workflow_dispatch:
+
+jobs:
+  run-build:
+    permissions:
+      packages: write
+    name: Run
+    uses: ./.github/workflows/maven_test.yml
+    if: github.repository == 'apache/spark'
+    with:
+      branch: branch-4.0
+      java: 21

.github/workflows/build_branch40_non_ansi.yml

@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Build / Non-ANSI (branch-4.0, Hadoop 3, JDK 17, Scala 2.13)"
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+
+jobs:
+  run-build:
+    permissions:
+      packages: write
+    name: Run
+    uses: ./.github/workflows/build_and_test.yml
+    if: github.repository == 'apache/spark'
+    with:
+      java: 17
+      branch: branch-4.0
+      hadoop: hadoop3
+      envs: >-
+        {
+          "PYSPARK_IMAGE_TO_TEST": "python-311",
+          "PYTHON_TO_TEST": "python3.11",
+          "SPARK_ANSI_SQL_MODE": "false",
+        }
+      jobs: >-
+        {
+          "build": "true",
+          "docs": "true",
+          "pyspark": "true",
+          "sparkr": "true",
+          "tpcds-1g": "true",
+          "docker-integration-tests": "true",
+          "yarn": "true"
+        }