Skip to content

Commit

Permalink
[add][plugin][icebergwriter] add icebergwriter code
Browse files Browse the repository at this point in the history
  • Loading branch information
awol2005ex committed Feb 20, 2025
1 parent dd744a7 commit a9b2fe2
Show file tree
Hide file tree
Showing 9 changed files with 1,423 additions and 0 deletions.
56 changes: 56 additions & 0 deletions docs/assets/jobs/icebergwriter.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
{
"job": {
"setting": {
"speed": {
"channel": 3
},
"errorLimit": {
"record": 0,
"percentage": 0
}
},
"content": [
{
"reader": {
"name": "rdbmsreader",
"parameter": {
"username": "root",
"password": "root",
"column": [
"*"
],
"connection": [
{
"querySql": [
"select 1+0 id ,now() ts1,'test1' as name"
],
"jdbcUrl": [
"jdbc:mysql://localhost:3306/ruoyi_vue_camunda?allowPublicKeyRetrieval=true"
]
}
],
"fetchSize": 1024
}
},
"writer": {
"name": "icebergwriter",
"parameter": {
"tableName": "test.test1",
"writeMode": "truncate",
"catalogType":"hadoop",
"warehouse": "s3a://pvc-91d1e2cd-4d25-45c9-8613-6c4f7bf0a4cc/iceberg",
"hadoopConfig": {

"fs.s3a.endpoint": "http://localhost:9000",
"fs.s3a.access.key": "gy0dX5lALP176g6c9fYf",
"fs.s3a.secret.key": "ReuUrCzzu5wKWAegtswoHIWV389BYl9AB1ZQbiKr",
"fs.s3a.connection.ssl.enabled": "false",
"fs.s3a.path.style.access": "true",
"fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem"
}
}
}
}
]
}
}
11 changes: 11 additions & 0 deletions package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,17 @@
<fileMode>0644</fileMode>
<outputDirectory>addax-${project.version}</outputDirectory>
</fileSet>
<fileSet>
<directory>plugin/writer/icebergwriter/target/icebergwriter-${project.version}/</directory>
<includes>
<include>**/*.*</include>
</includes>
<excludes>
<exclude>**/addax-common*.jar</exclude>
</excludes>
<fileMode>0644</fileMode>
<outputDirectory>addax-${project.version}</outputDirectory>
</fileSet>
<fileSet>
<directory>plugin/writer/influxdbwriter/target/influxdbwriter-${project.version}/</directory>
<includes>
Expand Down
37 changes: 37 additions & 0 deletions plugin/writer/icebergwriter/package.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<assembly
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-component-1.1.2.xsd">
<id>release</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/resources</directory>
<includes>
<include>*.json</include>
</includes>
<outputDirectory>plugin/writer/${project.artifactId}</outputDirectory>
</fileSet>
<fileSet>
<directory>target/</directory>
<includes>
<include>${project.artifactId}-${project.version}.jar</include>
</includes>
<outputDirectory>plugin/writer/${project.artifactId}</outputDirectory>
</fileSet>
</fileSets>

<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>plugin/writer/${project.artifactId}/libs</outputDirectory>
<scope>runtime</scope>
<excludes>
<exclude>com.wgzhao.addax:*</exclude>
</excludes>
</dependencySet>
</dependencySets>
</assembly>
Loading

0 comments on commit a9b2fe2

Please sign in to comment.