build.gradle

/*
 * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

buildscript {
    dependencies {
        classpath group: 'com.google.guava', name: 'guava', version: '28.1-jre'
        classpath 'org.ajoberstar:gradle-git:0.2.3'
    }
}

plugins {
    id 'java'
    id 'checkstyle'
    id 'application'
    id 'maven-publish'
    id 'com.google.protobuf' version '0.8.8'
    id 'jacoco'
    id 'idea'
    id 'com.github.spotbugs' version '4.0.0'
    id "de.undercouch.download" version "4.0.4"
}

application {
    mainClassName = 'com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp'
    applicationDefaultJvmArgs = ['-Xms64M',
                                 '-Xmx64M',
                                 '-XX:+UseSerialGC',
                                 '-XX:CICompilerCount=1',
                                 '-XX:-TieredCompilation',
                                 '-XX:InitialCodeCacheSize=4096',
                                 '-XX:InitialBootClassLoaderMetaspaceSize=30720',
                                 '-XX:MaxRAM=400m']
}

// Include pa_config folder in the distribution.
applicationDistribution.from(".") {
    include 'pa_config/*'
    include 'pa_bin/*'
}

distributions {
    main {
        contents {
            eachFile {
                it.path = it.path.replace("-$version/", '/')
            }
        }
    }
}

distZip {
    archiveName 'performance-analyzer-rca.zip'
}

publishing {
    publications {
        maven(MavenPublication) {
            groupId = 'com.amazon.opendistro.elasticsearch'
            artifactId = 'performanceanalyzer-rca'
            version = '1.3'
            from components.java
        }
    }
}

ext {
    opendistroVersion = '1.3'
    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
}

checkstyle {
    toolVersion '8.24'
    configFile file("config/checkstyle/checkstyle.xml")
}

tasks.withType(Checkstyle) {
    reports {
        xml.enabled false
        html.enabled true
        html.stylesheet resources.text.fromFile('config/xsl/severity-sorted.xsl')
    }
}

spotbugsMain {
    excludeFilter = file("checkstyle/findbugs-exclude.xml")
    effort = 'max'
    ignoreFailures = false

    reports {
        xml.enabled = false
        html.enabled = true
    }
}

spotbugsTest {
    ignoreFailures = true
}

jacoco {
    toolVersion = "0.8.5"
}

jacocoTestReport {
    reports {
        xml.enabled false
        html.enabled true
        csv.enabled false
    }

    afterEvaluate {
        classDirectories.from = files(classDirectories.files.collect {
            fileTree(dir: it,
                    include: [
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/**'
                    ],
                    exclude: [
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/grpc/**',
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/metrics/**',
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/net/requests**'
                    ])
        })
    }
}

jacocoTestCoverageVerification {
    afterEvaluate {
        classDirectories.from = files(classDirectories.files.collect {
            fileTree(dir: it,
                    include: [
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/**',
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/**'
                    ],
                    exclude: [
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/grpc/**'])
        })
    }
    violationRules {
        rule {
            limit {
                minimum = 0.8
            }
        }
    }
}

// to run coverage verification during the build (and fail when appropriate)
check.dependsOn jacocoTestCoverageVerification


version = "${opendistroVersion}.0"
if (isSnapshot) {
    version += "-SNAPSHOT"
}

test {
    enabled = true

}

task rcaTest(type: Test) {
    dependsOn checkstyleMain
    dependsOn checkstyleTest
    useJUnit {
        includeCategories 'com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.GradleTaskForRca'
    }
    testLogging.showStandardStreams = true
}

sourceCompatibility = 1.8
targetCompatibility = 1.8

compileJava {
    JavaVersion targetVersion = JavaVersion.toVersion(targetCompatibility);
    if (targetVersion.isJava9Compatible()) {
        options.compilerArgs += ["--add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED"]
    }
}

javadoc {
    options.addStringOption("-add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED")
}

project.afterEvaluate {
    JavaVersion targetVersion = JavaVersion.toVersion(targetCompatibility)

    // cannot contain the first version
    // should be '8' or '11' etc.
    String version = targetVersion.toString()
    if (version.length() > 2) {
        version = targetVersion.toString().substring(2)
    }

    compileJava.options.compilerArgs.removeAll(['--release', version])
}

repositories {
    mavenCentral()
}


configurations {
    includeJars
}

tasks.withType(JavaCompile) {
    options.warnings = false
}

dependencies {
    if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
        compile files("${System.properties['java.home']}/../lib/tools.jar")
    }

    compile 'org.jooq:jooq:3.10.8'
    compile 'org.bouncycastle:bcprov-jdk15on:1.60'
    compile 'org.bouncycastle:bcpkix-jdk15on:1.60'
    compile 'org.xerial:sqlite-jdbc:3.8.11.2'
    compile 'com.google.guava:guava:28.1-jre'
    compile 'com.fasterxml.jackson.core:jackson-annotations:2.8.11'
    compile 'com.fasterxml.jackson.core:jackson-databind:2.8.11'
    compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.13.0'
    compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.13.0'
    compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
    compile group: 'commons-io', name: 'commons-io', version: '2.3'
    implementation 'io.grpc:grpc-netty-shaded:1.25.0'
    implementation 'io.grpc:grpc-protobuf:1.25.0'
    implementation 'io.grpc:grpc-stub:1.25.0'
    implementation 'javax.annotation:javax.annotation-api:1.3.2'

    // JDK9+ has to run powermock 2+. https://github.com/powermock/powermock/issues/888
    testCompile group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-module-junit4', version: '2.0.0'
    testCompile group: 'org.mockito', name: 'mockito-core', version: '2.23.0'
    testCompile group: 'org.powermock', name: 'powermock-core', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-api-support', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-module-junit4-common', version: '2.0.0'
    testCompile group: 'org.javassist', name: 'javassist', version: '3.24.0-GA'
    testCompile group: 'org.powermock', name: 'powermock-reflect', version: '2.0.0'
    testCompile group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.3'
    testCompile group: 'org.objenesis', name: 'objenesis', version: '3.0.1'
    testCompile 'junit:junit:4.12'
    testCompile 'org.hamcrest:hamcrest:2.1'
    testCompile 'org.hamcrest:hamcrest-library:2.1'
}

protobuf {
    protoc {
        artifact = "com.google.protobuf:protoc:3.10.0"
    }

    plugins {
        grpc {
            artifact = 'io.grpc:protoc-gen-grpc-java:1.25.0'
        }

    }
    generateProtoTasks {
        all()*.plugins {
            grpc {}
        }
    }
}

idea {
    module {
        sourceDirs += file("${projectDir}/build/generated/source/proto/main/java");
        sourceDirs += file("${projectDir}/build/generated/source/proto/main/grpc");
    }
}


import org.ajoberstar.gradle.git.tasks.GitClone

import java.nio.file.Paths

String paDir
String dockerBuildDir
String dockerArtifactsDir

task cloneGitRepo(type: GitClone) {
    dockerBuildDir = Paths.get(getProject().getBuildDir().toString(), "docker-build").toString()
    mkdir dockerBuildDir
    if (localPaDir != null && !localPaDir.empty) {
        paDir = localPaDir
    } else {
        paDir = Paths.get(dockerBuildDir, "pa").toString()
    }

    def destination = file(paDir)
    uri = "https://github.com/opendistro-for-elasticsearch/performance-analyzer.git"
    destinationPath = destination
    bare = false
    enabled = !destination.exists() //to clone only once
}

task removeLicenses(type: Delete) {
    dependsOn(cloneGitRepo)
    def toDelete = Paths.get(paDir, 'licenses', 'performanceanalyzer-rca-1.3.jar.sha1')
    delete(toDelete)
}

task regenerateLicenses(type: Exec) {
    dependsOn(removeLicenses)
    workingDir(paDir)
    commandLine './gradlew', 'updateShas'
}

task buildPa(type: Exec) {
    dependsOn(assemble, publishToMavenLocal, regenerateLicenses)
    workingDir paDir
    println String.format('pa in dir: (%s) will be built.', paDir)
    commandLine './gradlew', 'assemble'
    println "PA repo located at '" + paDir + "' will be used."
}

task copyAllArtifacts(type: Copy) {
    dependsOn(buildPa)
    def projectPathStr = getProjectDir().toPath().toString()
    def dockerArtifacts = Paths.get(projectPathStr, 'docker').toString()
    def rcaArtifacts = Paths.get(projectPathStr, 'build', 'distributions', 'performance-analyzer-rca.zip')
    def paArtifacts = Paths.get(paDir, 'build', 'distributions', 'opendistro_performance_analyzer-1.3.0.0-SNAPSHOT.zip')

    dockerArtifactsDir = Paths.get(dockerBuildDir, 'rca-docker')
    from(dockerArtifacts)
    from(rcaArtifacts)
    from(paArtifacts)
    into(dockerArtifactsDir)
}

task copyReaderMetricsFiles(type: Copy) {
    dependsOn(buildPa)
    copy{
        from ('src/test/resources/reader'){
        }
        into 'build/private/test_resources'
    }
}

task buildDocker(type: Exec) {
    dependsOn(copyAllArtifacts)

    workingDir(dockerArtifactsDir)
    commandLine 'docker', 'build', '-t', 'odfe-es/pa-rca:1.0', '.'
}

task runDocker(type: Exec) {
    dependsOn(buildDocker)

    workingDir(dockerArtifactsDir)

    environment 'DATA_VOLUME1', 'esdata1'
    environment 'DATA_VOLUME2', 'esdata2'

    commandLine('docker-compose',
            '-f', 'docker-compose.yml',
            '-f', 'docker-compose.hostports.yml',
            '-f', 'docker-compose.cluster.yml',
            'up', 'elasticsearch1', 'elasticsearch2')

}

def printCommandOutput(proc) {
    Thread.start {
        BufferedReader stdInput = new BufferedReader(new
                InputStreamReader(proc.getInputStream()));

        BufferedReader stdError = new BufferedReader(new
                InputStreamReader(proc.getErrorStream()));

        System.out.println("Here is the standard output of the command:\n");
        String s = null;
        while ((s = stdInput.readLine()) != null) {
            System.out.println(s);
        }

        System.out.println("Here is the standard error of the command (if any):\n");
        while ((s = stdError.readLine()) != null) {
            System.out.println(s);
        }
    }
    int exitVal = proc.waitFor()
    System.out.println("Process exitValue: " + exitVal);
}

def runCommand(commandArr, envArr, workingDir, printOutput) {
    try {
        Process proc = Runtime.getRuntime().exec(commandArr, envArr, file(workingDir))
        if (printOutput) {
            printCommandOutput(proc)
        }
    } catch (Throwable th) {
        th.printStackTrace()
    }
}

def runInProcess(commandArr) {
    Process proc = new ProcessBuilder(commandArr).start();
    printCommandOutput(proc)
}

def runDockerCompose = { executionPath ->
    String[] commandArray = ['docker-compose',
                             '-f', 'docker-compose.yml',
                             '-f', 'docker-compose.hostports.yml',
                             '-f', 'docker-compose.cluster.yml',
                             'up', 'elasticsearch1', 'elasticsearch2']
    String[] env = ['DATA_VOLUME1=esdata1',
                    'DATA_VOLUME2=esdata2']
    runCommand(commandArray, env, file(executionPath), true)
}

def getHttpResponse(server, timeoutSeconds) {
    URL url = new URL(server)
    long timeout = timeoutSeconds * 1000
    long increments = 1000
    long passed = 0
    println 'waiting for ' + server
    while (passed < timeout) {
        try {
            HttpURLConnection con = (HttpURLConnection) url.openConnection()
            con.setRequestMethod("GET")
            InputStreamReader isr = new InputStreamReader(con.getInputStream())
            BufferedReader bufR = new BufferedReader(isr)
            String inputLine;
            StringBuffer content = new StringBuffer()
            while ((inputLine = bufR.readLine()) != null) {
                content.append(inputLine);
            }
            println(content.toString())
            bufR.close()
            break
        } catch (IOException) {
            Thread.sleep(increments);
            passed += increments
        }
    }
    if (passed == timeout) {
        throw new GradleException(String.format("ERROR:  %s not ready.", server));
    }
}

Boolean esIsUp

def esUpChecker = {
    String server = "http://localhost:9200"
    int timeoutSeconds = 5 * 60
    getHttpResponse(server, timeoutSeconds)
    esIsUp = true
}

Thread esRunnerThread

def runInThread(cl, input) {
    return Thread.start {
        cl(input)
    }
}

task waitForES {
    dependsOn(buildDocker)
    doLast {
        esRunnerThread = runInThread(runDockerCompose, dockerArtifactsDir)
        esUpChecker()
    }
}

String perfTopDir
String perfTopZip
String perfTopBin

task downloadPerfTop(type: Download) {
    String zip = 'perf-top-1.3.0.0-MACOS.zip'
    perfTopDir = Paths.get(dockerBuildDir, 'perftop').toString()
    mkdir perfTopDir

    src 'https://d3g5vo6xdbdb9a.cloudfront.net/downloads/perftop/' + zip
    dest perfTopDir
    overwrite false

    perfTopZip = Paths.get(perfTopDir, zip).toString()
    File f = file(perfTopZip)
    enabled = !f.exists() //to clone only once
}

task unzipPerfTop(type: Copy) {
    dependsOn(downloadPerfTop)

    perfTopBin = Paths.get(perfTopDir, 'perf-top-macos').toString()

    def zipFile = file(perfTopZip)
    def outputDir = file(perfTopDir)

    from zipTree(zipFile)
    into outputDir

    enabled = !file(perfTopBin).exists()
}

task enablePaAndRca(type: Exec) {
    dependsOn(waitForES)
    commandLine('curl', 'localhost:9200/_opendistro/_performanceanalyzer/cluster/config', '-H',
            'Content-Type: application/json', '-d', '{"enabled": true}')
    commandLine('curl', 'localhost:9200/_opendistro/_performanceanalyzer/rca/cluster/config',
            '-H', 'Content-Type: application/json', '-d', '{"enabled": true}')
}

task runPerftop {
    dependsOn(enablePaAndRca, unzipPerfTop)
    doLast {
        println '============================================================================'
        println 'invoke perftop as: '
        println String.format("%s --dashboard %s", Paths.get(perfTopBin).toString(), 'ClusterOverview')
        println '============================================================================'
    }
}

task generateLoad {
    dependsOn(enablePaAndRca, runPerftop)

    doLast {
        String[] deleteIndex = ['curl', '-X', 'DELETE', 'localhost:9200/accounts?pretty']
        String[] bulk = ['curl', '-s', '-H', "Content-Type: application/x-ndjson",
                         '-XPOST', 'localhost:9200/_bulk', '--data-binary', '@accounts.json']
        String[] search = ['curl', '-X', 'GET', 'localhost:9200/_search?pretty', '-H',
                           'Content-Type:application/json', '-d',
                           '{"query": {"range": {"balance": {"gte": 100, "lte": 100000}}}}']

        String wdir = Paths.get(getProjectDir().getAbsolutePath(), 'src', 'test', 'resources')
                .toString()
        while (true) {
            // runCommand(deleteIndex, null, wdir, false)
            //println("starting indexing: " + System.currentTimeMillis())
            runCommand(bulk, null, wdir, false)
            //println("completed indexing: " + System.currentTimeMillis())
            Thread.sleep(1000)
            //println("starting search" + System.currentTimeMillis())
            runCommand(search, null, wdir, false)
            Thread.sleep(300)
            //println("completed search" + System.currentTimeMillis())
            runCommand(deleteIndex, null, wdir, false)
            Thread.sleep(1000)
        }
    }
}

task runRally {
    dependsOn(enablePaAndRca, runPerftop)

    doLast {
        String rallyTrack='pmc'

        String[] esrally = ['docker', 'run', '--network=host', 'elastic/rally',
                                '--track=' + rallyTrack, '--test-mode', '--pipeline=benchmark-only',
                '--target-hosts=localhost:9200']

        String wdir = Paths.get(getProjectDir().getAbsolutePath(), 'src', 'test', 'resources')
                .toString()
        while (true) {
            runCommand(esrally, null, wdir, true)
            println('Get all RCAs using:')
            println('curl localhost:9600/_opendistro/_performanceanalyzer/rca?all | python -m json.tool')
            Thread.sleep(1000)
        }
    }
}