build.gradle

/*
 * Copyright 2018 LinkedIn Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
buildscript {
  repositories {
    mavenCentral()
    maven {
      url 'https://plugins.gradle.org/m2/'
    }
  }
  dependencies {
    classpath 'com.cinnober.gradle:semver-git:2.2.3'
    classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.0.14'
  }
}

plugins {
  id 'com.gradle.build-scan' version '3.2'
  id 'com.github.kt3k.coveralls' version '2.6.3'
  id 'jacoco'
  id 'idea'
}

// https://docs.gradle.com/enterprise/gradle-plugin/#connecting_to_scans_gradle_com
gradleEnterprise {
  buildScan {
    termsOfServiceUrl = "https://gradle.com/terms-of-service"
    termsOfServiceAgree = "yes"
  }
}

apply plugin: 'com.cinnober.gradle.semver-git'

apply plugin: 'compare-gradle-builds'

compareGradleBuilds {
  targetBuild.gradleVersion = "4.0"
}

allprojects {
  apply plugin: 'jacoco'

  repositories {
    mavenCentral()
    mavenLocal()
//  need this for rest.li/pegasus 28.* artifacts until they are in Maven Central:
    maven {
      url 'https://linkedin.jfrog.io/artifactory/open-source/'
    }
  }
}

ext.versions = [hadoop: '2.10.0',
                hive  : '1.1.0',
                pig   : '0.11.0',
                restli: '29.16.0',
                slf4j : '1.7.18',]

ext.deps = [
    // External dependencies
    // restli 1.* had this one included transitively. restli 28.* doesn't. But we still need it:
    json                 : 'org.json:json:20170516',
    assertj              : 'org.assertj:assertj-core:3.8.0',
    awaitility           : 'org.awaitility:awaitility:3.0.0',
    collections          : 'commons-collections:commons-collections:3.2.2',
    commonsCli           : 'commons-cli:commons-cli:1.3.1',
    commonsLang          : 'commons-lang:commons-lang:2.6',
    commonsCodec         : 'commons-codec:commons-codec:1.13',
    commonsCompress      : 'org.apache.commons:commons-compress:1.16.1',
    dbcp2                : 'org.apache.commons:commons-dbcp2:2.7.0',
    dbutils              : 'commons-dbutils:commons-dbutils:1.5',
    fileupload           : 'commons-fileupload:commons-fileupload:1.2.1',
    gson                 : 'com.google.code.gson:gson:2.8.1',
    guava                : 'com.google.guava:guava:21.0',
    guice                : 'com.google.inject:guice:4.1.0',
    h2                   : 'com.h2database:h2:1.4.193',
    hadoopAnnotations    : "org.apache.hadoop:hadoop-annotations:" + versions.hadoop,
    hadoopAuth           : "org.apache.hadoop:hadoop-auth:" + versions.hadoop,
    hadoopCommon         : "org.apache.hadoop:hadoop-common:" + versions.hadoop,
    hadoopHdfs           : "org.apache.hadoop:hadoop-hdfs:" + versions.hadoop,
    hadoopMRClientCommon : "org.apache.hadoop:hadoop-mapreduce-client-common:" + versions.hadoop,
    hadoopMRClientCore   : "org.apache.hadoop:hadoop-mapreduce-client-core:" + versions.hadoop,
    hiveCli              : "org.apache.hive:hive-cli:" + versions.hive,
    hiveExecCore         : "org.apache.hive:hive-exec:" + versions.hive + ":core",
    hiveMetastore        : "org.apache.hive:hive-metastore:" + versions.hive,
    httpclient           : 'org.apache.httpcomponents:httpclient:4.5.3',
    io                   : 'commons-io:commons-io:2.4',
    jacksonCoreAsl       : 'org.codehaus.jackson:jackson-core-asl:1.9.5',
    jacksonMapperAsl     : 'org.codehaus.jackson:jackson-mapper-asl:1.9.5',
    javaxElApi           : 'javax.el:javax.el-api:3.0.0',
    javaxEl              : 'org.glassfish:javax.el:3.0.0',
    jetty                : 'org.mortbay.jetty:jetty:6.1.26',
    jettyUtil            : 'org.mortbay.jetty:jetty-util:6.1.26',
    jexl                 : 'org.apache.commons:commons-jexl:2.1.1',
    jodaTime             : 'joda-time:joda-time:2.0',
    jopt                 : 'net.sf.jopt-simple:jopt-simple:5.0.3',
    jsr305               : 'com.google.code.findbugs:jsr305:3.0.2',
    junit                : 'junit:junit:4.12',
    kafkaLog4jAppender   : 'org.apache.kafka:kafka-log4j-appender:0.10.0.0',
    k8sClient            : 'io.kubernetes:client-java:15.0.1',
    k8sClientFluent      : 'io.kubernetes:client-java-api-fluent:15.0.1',
    log4j                : 'log4j:log4j:1.2.16',
    mail                 : 'com.sun.mail:javax.mail:1.6.2',
    math3                : 'org.apache.commons:commons-math3:3.0',
    metricsCore          : 'io.dropwizard.metrics:metrics-core:3.2.6',
    metricsJvm           : 'io.dropwizard.metrics:metrics-jvm:3.2.6',
    mockito              : 'org.mockito:mockito-core:2.28.2',
    mongoDriver          : 'org.mongodb:mongo-java-driver:2.14.0',
    mysqlConnector       : 'mysql:mysql-connector-java:8.0.20',
    pig                  : 'org.apache.pig:pig:' + versions.pig,
    parquetAvro          : 'com.twitter:parquet-avro:1.6.0',
    parquetBundle        : 'com.twitter:parquet-hadoop-bundle:1.3.2',
    powermock            : 'org.powermock:powermock-api-mockito2:2.0.2',
    powermockmodulejunit4: 'org.powermock:powermock-module-junit4:2.0.2',
    quartz               : 'org.quartz-scheduler:quartz:2.2.1',
    restliGenerator      : 'com.linkedin.pegasus:generator:' + versions.restli,
    restliServer         : 'com.linkedin.pegasus:restli-server:' + versions.restli,
    restliTools          : 'com.linkedin.pegasus:restli-tools:' + versions.restli,
    slf4j                : 'org.slf4j:slf4j-api:' + versions.slf4j,
    slf4jLog4j           : 'org.slf4j:slf4j-log4j12:' + versions.slf4j,
    sparkCore            : 'org.apache.spark:spark-core_2.12:3.3.0',
    springTest           : 'org.springframework:spring-test:3.2.18.RELEASE',
    snakeyaml            : 'org.yaml:snakeyaml:1.18',
    velocity             : 'org.apache.velocity:velocity-engine-core:2.3',
    velocityTools        : 'org.apache.velocity.tools:velocity-tools-generic:3.1',
    bcprov               : 'org.bouncycastle:bcprov-jdk15on:1.54',
    jacksonAnnotation    : 'com.fasterxml.jackson.core:jackson-annotations:2.7.4',
    jacksonDatabind      : 'com.fasterxml.jackson.core:jackson-databind:2.7.4',
    jasypt               : 'org.jasypt:jasypt:1.9.2',
    jacksonCore          : 'com.fasterxml.jackson.core:jackson-core:2.9.2',
    jsonassert           : 'org.skyscreamer:jsonassert:1.5.0',
    systemRules          : 'com.github.stefanbirkner:system-rules:1.19.0',
    javaxValidation      : 'javax.validation:validation-api:2.0.1.Final',
    hibernateValidator   : 'org.hibernate:hibernate-validator:6.1.5.Final',
    failsafe             : 'dev.failsafe:failsafe:3.2.4']

subprojects {
  apply plugin: 'java'
  apply plugin: 'net.ltgt.errorprone'

  // Set the same version for all sub-projects to root project version
  version = rootProject.version

  configurations.errorprone {
    dependencies {
      errorprone 'com.google.errorprone:error_prone_core:2.1.0'
    }
  }

  plugins.withType(JavaPlugin) {
    sourceCompatibility = JavaVersion.VERSION_1_8

    /*
     TODO remove afterEvaluate block
     After Evaluate block was added to do a lazy evaluation. This piece of code gets executed by gradle in the
     configuration phase. However, for some reason the version field was not updated by the LinkedIn build
     infrastructure. Thus, using afterEvaluate to do a lazy evaluation of this code block.
     More specifically afterEvaluate kicks in after the rest of the project is configured

     See: http://stackoverflow.com/questions/16218888/can-gradle-extensions-handle-lazy-evaluation-of-a-property
     See: http://stackoverflow.com/questions/16070567/difference-between-gradles-terms-evaluation-and-execution
     */
    project.afterEvaluate {
      // Set the Title and Version fields in the jar
      jar {
        manifest {
          attributes('Implementation-Title': project.name,
              'Implementation-Version': project.version)
        }
      }
    }

    dependencies {
      compile deps.log4j
      compile deps.guice
      compile deps.slf4j
      runtime deps.slf4jLog4j

      testCompile deps.assertj
      testCompile deps.junit
      testCompile deps.mockito
      testCompile deps.powermock
      testCompile deps.powermockmodulejunit4
      testCompile deps.jsonassert
      testCompile deps.awaitility
      testCompile deps.springTest
    }

    test {
      // Set maxParallelForks to a large number and let gradle to force it to a the
      // max-workers number when needed.
      // NOTE! For some reason anything more than 1 doesn't work well for azkaban build:
      // any maxParallelForks > 1 seems to make `./gradlew cleanTest test` ~4 times slower
      maxParallelForks = 1
    }
  }

  // Common distribution plugin settings for sub-modules
  plugins.withType(DistributionPlugin) {
    distTar {
      compression = Compression.GZIP
      extension = 'tar.gz'
    }
  }

  /**
   * Terminate compilation if warnings occur.*/
  tasks.withType(JavaCompile) {
    options.compilerArgs += ["-Werror"]
  }

  /**
   * Print test execution summary when informational logging is enabled.*/
  test {
    testLogging {
      exceptionFormat = 'full'
      afterSuite { desc, result ->
        if (desc.getParent()) {
          logger.info desc.getName()
        } else {
          logger.info "Overall"
        }
        logger.info "  ${result.resultType} (" + "${result.testCount} tests, " +
            "${result.successfulTestCount} passed, " + "${result.failedTestCount} failed, " +
            "${result.skippedTestCount} skipped)"
      }
    }
  }
}

/**
 * Gradle wrapper task.
 * https://docs.gradle.org/5.2.1/userguide/gradle_wrapper.html#customizing_wrapper
 * https://stackoverflow.com/questions/53709282/cannot-add-task-wrapper-as-a-task-with-that-name-already-exists
 **/
wrapper {
  distributionType('ALL')
}

if (System.env.TRAVIS == 'true') {
  allprojects {
    tasks.withType(Test) {
      // Reduce the memory pressure on the Travis CI to reduce build failures.
      maxParallelForks = 2
      minHeapSize = '512m'
      maxHeapSize = '4g'
      outputs.upToDateWhen { false }
    }
  }
}

def publishedProjects = subprojects.findAll()

// Aggregate all subproject reports into one.
task jacocoAggregateReport(type: JacocoReport, group: 'Coverage reports') {
  description = 'Generates an aggregate report from all subprojects'

  dependsOn(publishedProjects.test)

  // JacocoReport-type task requires users to specify which classes to be covered
  // See details in
  // https://docs.gradle.org/current/dsl/org.gradle.testing.jacoco.tasks.JacocoReport.html
  sourceDirectories = files(publishedProjects.sourceSets.main.allSource.srcDirs)
  classDirectories = files(publishedProjects.sourceSets.main.output)
  executionData = files(publishedProjects.jacocoTestReport.executionData)

  doFirst {
    executionData = files(executionData.findAll { it.exists() })
  }

  reports {
    html.enabled = true // human readable
    xml.enabled = true // required by coveralls
  }
}

coveralls {
  sourceDirs = publishedProjects.sourceSets.main.allSource.srcDirs.flatten()
  jacocoReportPath = "${buildDir}/reports/jacoco/jacocoAggregateReport/jacocoAggregateReport.xml"
}

tasks.coveralls {
  dependsOn jacocoAggregateReport
}