Skip to content

Commit

Permalink
[INLONG-10725][CI] Add UT test workflow for flink 1.18 connectors. (#…
Browse files Browse the repository at this point in the history 
…10726)
  • Loading branch information
@XiaoYou201
XiaoYou201 authored Sep 2, 2024
1 parent 6234b9e commit 5fd7cc0
Show file tree
Hide file tree
Showing 13 changed files with 1,081 additions and 39 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci_ut.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ jobs:
CI: false

- name: Unit test with Maven
run: mvn --batch-mode --update-snapshots -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13
run: mvn --batch-mode --update-snapshots -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13,!:sort-end-to-end-tests-v1.18
env:
CI: false

Expand Down
96 changes: 96 additions & 0 deletions .github/workflows/ci_ut_flink18.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name:
InLong Unit Test For Flink 1.18

on:
push:
paths:
- '.github/workflows/ci_ut_flink18.yml'
- 'inlong-sort/**'
- '!**.md'

pull_request:
paths:
- '.github/workflows/ci_ut_flink18.yml'
- 'inlong-sort/**'
- '!**.md'

jobs:
unit-test:
name: Unit Test
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v4

# Release space size
- name: Remove unnecessary packages
run: |
echo "=== Before pruning ==="
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache
echo "=== After pruning ==="
df -h
- name: Set up JDK
uses: actions/setup-java@v4
with:
java-version: 8
distribution: adopt

- name: Cache Maven packages
uses: actions/cache@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/inlong
key: ${{ runner.os }}-inlong-flink18-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-inlong-flink18

- name: Build for Flink 1.18 with Maven
run: mvn --update-snapshots -e -V clean install -U -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000
env:
CI: false

- name: Unit test for Flink 1.18 with Maven
run: mvn --update-snapshots -e -V verify -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18
env:
CI: false

- name: Upload unit test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: surefire-reports
path: ./**/target/surefire-reports/
if-no-files-found: ignore

- name: Upload integration test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: failsafe-reports
path: ./**/target/failsafe-reports/
if-no-files-found: ignore

- name: Clean up build packages
run: mvn clean
54 changes: 42 additions & 12 deletions inlong-sort/sort-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,18 +72,6 @@
<version>${mysql.jdbc.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-base</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-csv</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
Expand All @@ -106,6 +94,18 @@
<artifactId>flink-table-common</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-csv</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-base</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-flink-dependencies-${sort.flink.version}</artifactId>
Expand Down Expand Up @@ -251,6 +251,18 @@
<artifactId>flink-table-common</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-csv</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-base</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-flink-dependencies-${sort.flink.version}</artifactId>
Expand Down Expand Up @@ -372,6 +384,24 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-connector-jdbc-v1.18</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-connector-elasticsearch6-v1.18</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-connector-elasticsearch7-v1.18</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
Expand Down
75 changes: 50 additions & 25 deletions inlong-sort/sort-dist/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,31 +55,6 @@
<artifactId>sort-format-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-csv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-csv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-kv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-rowdata-kv</artifactId>
Expand Down Expand Up @@ -134,6 +109,31 @@
<artifactId>sort-format-json-v1.13</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-csv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-csv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-kv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-sql-parquet_${scala.binary.version}</artifactId>
Expand Down Expand Up @@ -172,6 +172,31 @@
<artifactId>sort-format-json-v1.15</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-csv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-csv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-inlongmsg-kv</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-sql-parquet</artifactId>
Expand Down
9 changes: 9 additions & 0 deletions inlong-sort/sort-end-to-end-tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,15 @@
<module>sort-end-to-end-tests-v1.15</module>
</modules>
</profile>
<profile>
<id>v1.18</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<modules>
<module>sort-end-to-end-tests-v1.18</module>
</modules>
</profile>
</profiles>

</project>
Loading

0 comments on commit 5fd7cc0

Please sign in to comment.