Refactor mantis-connector-iceberg 3rd party dependency specifications

first pass at refactor 3rd party dependency specifications for mantis-connector-iceberg and corresponding changes to root build.gradle (issue Netflix#345)
mabelbot · Mar 1, 2023 · 2d27da4 · 2d27da4
1 parent 05a20f3
commit 2d27da4
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 16 deletions.
diff --git a/build.gradle b/build.gradle
@@ -35,14 +35,17 @@ buildscript {
 }
 
 ext.versions = [
+        avroVersion : "1.10.2",
         flink    : "1.14.2",
         hadoop   : "2.7.7",
+        icebergVersion : "0.14.+",
         jctoolsVersion : "1.+",
         junit4   : "4.11",
         junit5   : "5.4.+",
         mockito  : "2.0.+",
         mockito3 : "3.+",
         nettyVersion : "4.1.34.Final",
+        parquetVersion : "1.12.0",
         snappyVersion : "1.1.+",
         spectator: "1.3.+",
         slf4j    : "1.7.0",
@@ -51,6 +54,7 @@ ext.versions = [
 
 ext.libraries = [
         asyncHttpClient: "org.asynchttpclient:async-http-client:2.12.3",
+        avro           : "org.apache.avro:avro:${versions.avroVersion}",
         commonsIo      : "commons-io:commons-io:2.11.0",
         commonsLang3   : 'org.apache.commons:commons-lang3:3.5',
         flinkCore      : "org.apache.flink:flink-core:${versions.flink}",
@@ -67,6 +71,10 @@ ext.libraries = [
         hadoopCommon   : "org.apache.hadoop:hadoop-common:${versions.hadoop}",
         hadoopS3       : "org.apache.hadoop:hadoop-aws:${versions.hadoop}",
         hamcrest       : "org.hamcrest:hamcrest-core:1.3",
+        icebergApi     : "org.apache.iceberg:iceberg-api:${versions.icebergVersion}",
+        icebergCore    : "org.apache.iceberg:iceberg-core:${versions.icebergVersion}",
+        icebergData    : "org.apache.iceberg:iceberg-data:${versions.icebergVersion}",
+        icebergParquet : "org.apache.iceberg:iceberg-parquet:${versions.icebergVersion}", 
         jcip           : "net.jcip:jcip-annotations:1.0",
         jctools        : "org.jctools:jctools-core:${versions.jctoolsVersion}",
         junitJupiter   : [
@@ -82,6 +90,7 @@ ext.libraries = [
         nettyBuffer    : "io.netty:netty-buffer:${versions.nettyVersion}",
         nettyCodec     : "io.netty:netty-codec-http:${versions.nettyVersion}",
         nettyEpoll     : "io.netty:netty-transport-native-epoll:${versions.nettyVersion}:linux-x86_64",
+        parquetHadoop  : "org.apache.parquet:parquet-hadoop:${versions.parquetVersion}",
         rxJava         : "io.reactivex:rxjava:1.3.8",
         rxNettyShaded  : "com.netflix:mantis-rxnetty:0.4.19.1",
         slf4jApi       : "org.slf4j:slf4j-api:${versions.slf4j}",

diff --git a/mantis-connectors/mantis-connector-iceberg/build.gradle b/mantis-connectors/mantis-connector-iceberg/build.gradle
@@ -17,12 +17,6 @@
 apply plugin: 'com.github.johnrengelman.shadow'
 apply plugin: 'mantis'
 
-ext {
-    hadoopVersion = '2.7.3'
-    icebergVersion = '0.14.+'
-    avroVersion = '1.10.2'
-    parquetVersion = '1.12.0'
-}
 
 dependencies {
     configurations {
@@ -33,30 +27,30 @@ dependencies {
             // we want a parquet version above 1.12.0 because we need this fix
             // https://issues.apache.org/jira/browse/PARQUET-1851
             resolutionStrategy {
-                force "org.apache.parquet:parquet-hadoop:${parquetVersion}"
+                force libraries.parquetHadoop
             }
         }
     }
     implementation project(":mantis-runtime")
     implementation libraries.slf4jApi
 
     // We only need the Configuration interface. Users can bring their own hadoop-common version.
-    shadow "org.apache.hadoop:hadoop-common:$hadoopVersion"
+    shadow libraries.hadoopCommon
 
     // Exclude in case there are differences in SHAs between 0.7.0 incubation versions.
-    shadow "org.apache.iceberg:iceberg-api:$icebergVersion"
-    shadow "org.apache.iceberg:iceberg-core:$icebergVersion"
-    shadow "org.apache.iceberg:iceberg-data:$icebergVersion"
-    shadow "org.apache.iceberg:iceberg-parquet:$icebergVersion"
-    shadow "org.apache.avro:avro:$avroVersion"
-    shadow "org.apache.parquet:parquet-hadoop:${parquetVersion}"
+    shadow libraries.icebergApi
+    shadow libraries.icebergCore
+    shadow libraries.icebergData
+    shadow libraries.icebergParquet
+    shadow libraries.avro
+    shadow libraries.parquetHadoop
 
     // We only need this for local mains(). Users bring their own implementation.
     shadow libraries.slf4jLog4j12
 
     testImplementation project(":mantis-runtime").sourceSets.test.output
-    testImplementation "org.apache.hadoop:hadoop-common:$hadoopVersion"
-    testImplementation "org.apache.iceberg:iceberg-data:$icebergVersion" 
+    testImplementation libraries.hadoopCommon
+    testImplementation libraries.icebergData 
     testImplementation libraries.junitJupiter
     testImplementation libraries.mockitoCore
     testImplementation libraries.spectatorApi