Fix kafka-connect-hive path, fix kafka-connect-hive project, fix grad…

…le version and more

Signed-off-by: Marios Andreopoulos <[email protected]>

gradle.properties

@@ -15,6 +15,6 @@
 #
 
 # Always set the version to -SNAPSHOT after a release
-version=1.3.0-SNAPSHOT
+version=1.2.0-SNAPSHOT
 ossrhUsername=you
 ossrhPassword=me

gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-all.zip

kafka-connect-hive/build.gradle

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2018 Datamountaineer.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+project(":kafka-connect-hive") {
+
+    ext {
+        parquetVersion = "1.8.3"
+        hiveVersion = "2.1.1"
+        joddVersion = "4.1.4"
+        hadoopVersion = "2.6.5"
+        catsVersion = "1.2.0"
+    }
+
+    dependencies {
+        compile("org.apache.parquet:parquet-column:$parquetVersion")
+        compile("org.apache.parquet:parquet-encoding:$parquetVersion")
+        compile("org.apache.parquet:parquet-hadoop:$parquetVersion")
+        compile("org.jodd:jodd-core:$joddVersion")
+        compile("org.apache.hive:hive-jdbc:$hiveVersion")
+        compile("org.apache.hive:hive-exec:$hiveVersion")
+        compile("org.apache.hadoop:hadoop-common:$hadoopVersion")
+        compile("org.apache.hadoop:hadoop-hdfs:$hadoopVersion")
+        compile("org.apache.hadoop:hadoop-mapreduce:$hadoopVersion")
+        compile("org.apache.hadoop:hadoop-mapreduce-client:$hadoopVersion")
+        compile("org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion")
+        compile("org.typelevel:cats-core_2.11:$catsVersion")
+
+        //testCompile 'org.apache.logging.log4j:log4j:2.11.0'
+    }
+}

kafka-connect-hive/it/build.gradle

@@ -13,75 +13,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-buildscript {
 
-    repositories {
-        jcenter()
-        maven {
-            url 'https://plugins.gradle.org/m2/'
-        }
-    }
-}
-
-ext {
-    scalaMajorVersion = '2.11'
-    scala = '2.11.12'
-    scalaCheck = '1.13.4'
-    scalaTest = '3.0.4'
-    scalaLoggingVersion = '2.1.2'
-    junit = '4.11'
-    parquetVersion = "1.8.3"
-    hiveVersion = "2.1.1"
-    joddVersion = "4.1.4"
-    hadoopVersion = "2.6.5"
-    catsVersion = "1.1.0"
-
-    // The following 4 need to align to compile against a particular version
-    confluentVersion = '4.1.0'
-    kafkaVersion = '1.1.0'
-    dataMountaineerCommonVersion = "1.1.5"
-    link4jVersion = "1.8.0"
-}
-
-group = 'com.datamountaineer'
-version = project.version
-description = "stream-reactor"
-
-apply plugin: 'java'
-apply plugin: 'scala'
-
-sourceCompatibility = 1.8
-targetCompatibility = 1.8
-
-repositories {
-    mavenLocal()
-    mavenCentral()
-    maven { url "http://packages.confluent.io/maven/" }
-}
+project(":kafka-connect-hive:it") {
 
-dependencies {
-
-    testCompile "org.scala-lang:scala-library:$scala"
-    testCompile "org.scala-lang:scala-compiler:$scala"
-    testCompile "com.typesafe.scala-logging:scala-logging-slf4j_$scalaMajorVersion:$scalaLoggingVersion"
-
-    testCompile ("org.apache.kafka:connect-json:$kafkaVersion")
-    testCompile "io.confluent:kafka-connect-avro-converter:$confluentVersion"
-    testCompile "io.confluent:kafka-schema-registry:$confluentVersion"
-
-    testCompile 'org.asynchttpclient:async-http-client:2.5.2'
-    testCompile "org.scalatest:scalatest_$scalaMajorVersion:$scalaTest"
-    testCompile 'org.apache.logging.log4j:log4j:2.11.0'
-
-    testCompile("org.apache.hive:hive-jdbc:$hiveVersion")
-    testCompile project(":kafka-connect-hive:connector")
+    ext {
+        parquetVersion = "1.8.3"
+        hiveVersion = "2.1.1"
+        joddVersion = "4.1.4"
+        hadoopVersion = "2.6.5"
+        catsVersion = "1.1.0"
+    }
 
-    testRuntime 'org.pegdown:pegdown:1.1.0'
-}
+    dependencies {
+        testCompile 'org.asynchttpclient:async-http-client:2.5.2'
+        testCompile 'org.apache.logging.log4j:log4j:2.11.0'
 
+        testCompile("org.apache.hive:hive-jdbc:$hiveVersion")
+        testCompile project(":kafka-connect-hive")
+    }
 
-test {
-    onlyIf {
-        project.hasProperty("integration-tests")
+    test {
+        onlyIf {
+            project.hasProperty("integration-tests")
+        }
     }
 }

kafka-connect-hive/it/docker-compose.yml

@@ -13,7 +13,7 @@ services:
       CONNECTORS: "redis"
       ADV_HOST: "127.0.0.1"
     volumes:
-      - ../connector/build/libs/:/connectors/hive
+      - ../build/libs/:/connectors/hive
 
   namenode:
     image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8

settings.gradle

@@ -25,7 +25,7 @@ include 'kafka-connect-bloomberg',
         'kafka-connect-hbase',
         'kafka-connect-redis',
         'kafka-connect-hazelcast',
-        'kafka-connect-hive:connector',
+        'kafka-connect-hive',
         'kafka-connect-hive:it',
         'kafka-connect-rethink',
         'kafka-connect-jms',