build.gradle

apply plugin: 'base'

description = 'Builds the Machine Learning native binaries'

import org.elastic.gradle.UploadS3Task
import org.gradle.internal.os.OperatingSystem
import org.gradle.plugins.ide.eclipse.model.SourceFolder
import org.gradle.util.DistributionLocator
import org.gradle.util.GradleVersion

import java.util.zip.ZipFile

import static org.gradle.api.tasks.wrapper.Wrapper.DistributionType

String versionQualifier = System.getProperty("build.version_qualifier", "")
boolean isSnapshot = "true".equals(System.getProperty("build.snapshot", "true"))
String mlDebug = System.getProperty("build.ml_debug", "")

boolean isWindows = OperatingSystem.current().isWindows()
boolean isLinux = OperatingSystem.current().isLinux()
boolean isMacOsX = OperatingSystem.current().isMacOsX()

allprojects {
  group = 'org.elasticsearch.ml'
  version = elasticsearchVersion
  if (versionQualifier != "") {
    version += '-' + versionQualifier
  }
  if (isSnapshot) {
    version += '-SNAPSHOT'
  }
}

// Secret key and access key are only needed when
// using the upload tasks near the end of this file.
String envMlAwsAccessKey = System.env.ML_AWS_ACCESS_KEY
if (envMlAwsAccessKey != null) {
  project.ext.mlAwsAccessKey = envMlAwsAccessKey
} else if (project.hasProperty('ML_AWS_ACCESS_KEY')) {
  project.ext.mlAwsAccessKey = ML_AWS_ACCESS_KEY
}

String envMlAwsSecretKey = System.env.ML_AWS_SECRET_KEY
if (envMlAwsSecretKey != null) {
  project.ext.mlAwsSecretKey = envMlAwsSecretKey
} else if (project.hasProperty('ML_AWS_SECRET_KEY')) {
  project.ext.mlAwsSecretKey = ML_AWS_SECRET_KEY
}

String cppCrossCompile = System.env.CPP_CROSS_COMPILE
if (cppCrossCompile == null) {
  if (project.hasProperty('CPP_CROSS_COMPILE')) {
    cppCrossCompile = CPP_CROSS_COMPILE
  } else {
    cppCrossCompile = ''
  }
}
if (cppCrossCompile != '' && cppCrossCompile != 'macosx' && cppCrossCompile != 'aarch64') {
  throw new GradleException("CPP_CROSS_COMPILE property must be empty, 'macosx' or 'aarch64'")
}

String osArch = System.properties['os.arch']
// Some versions of Java report hardware architectures that
// don't match other tools - these need to be normalized
if (osArch == 'amd64') {
  osArch = 'x86_64'
} else if (osArch == 'arm64') {
  osArch = 'aarch64'
}
String artifactClassifier
if (isWindows) {
  artifactClassifier = 'windows-x86_64'
} else if (isMacOsX || cppCrossCompile == 'macosx') {
  artifactClassifier = 'darwin-' + osArch
} else if (cppCrossCompile != '') {
  artifactClassifier = 'linux-' + cppCrossCompile
} else {
  artifactClassifier = 'linux-' + osArch
}

// Ensure we have a consistent combination of build config type and build directory
// based on whether we want a debug build or not.
// By default a "RelWithDebInfo" build is performed.
// Note that we are catering for single-config CMake generators - such as 'Unix Makefiles' -
// where the build type is specified at configuration time, as well as multi-config generators - such as Visual Studio -
// where the desired build type is specified at build time with the '--config' option. (It's safe to always specify the
// '--config' option as it will simply be ignored by a single-config generated build system)
String cmakeFlags = '--no-warn-unused-cli -D CMAKE_TOOLCHAIN_FILE=cmake/' + artifactClassifier + '.cmake'
project.ext.cmakeBuildDir = "cmake-build-relwithdebinfo"
project.ext.cmakeBuildType = "RelWithDebInfo"
if (mlDebug.toBoolean()) {
  project.ext.cmakeBuildDir = "cmake-build-debug"
  project.ext.cmakeBuildType = "Debug"
}

String msystem = "$System.env.MSYSTEM"
if (isWindows && msystem != 'MINGW64') {
  project.ext.shell = "cmd.exe"
  project.ext.setEnv = ".\\set_env.bat"
  project.ext.shellFlag = "/c"
  project.ext.cleanCmd = "if exist ${cmakeBuildDir}\\ (rmdir /s /q ${cmakeBuildDir}) && if exist 3rd_party\\eigen (rmdir /s /q 3rd_party\\eigen)"
  // Stripping the binaries is not necessary on Windows, execute a no-op command instead
  project.ext.stripCmd = "cd ."
} else {
  // Use the bash shell for the C++ build (Git bash when on Windows using MinGW)
  project.ext.shell = isWindows ? "C:\\Program Files\\Git\\bin\\bash" : "/bin/bash"
  project.ext.setEnv = "source ./set_env.sh"
  project.ext.shellFlag = "-c"
  project.ext.cleanCmd = "rm -rf ${cmakeBuildDir} 3rd_party/eigen"
  // Stripping the binaries is not necessary on Windows, execute a no-op command instead
  project.ext.stripCmd = isWindows ? 'true' : "dev-tools/strip_binaries.sh"
}

project.ext.numCpus = Runtime.runtime.availableProcessors()
project.ext.makeEnvironment = [ 'CPP_CROSS_COMPILE': cppCrossCompile,
                                'VERSION_QUALIFIER': versionQualifier,
                                'SNAPSHOT': (isSnapshot ? 'yes' : 'no'),
                                'ML_DEBUG': mlDebug ]

configurations.all {
  // Check for updates every build
  resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
}

clean {
  doLast {
    exec {
      environment makeEnvironment
      commandLine shell
      args shellFlag, "${setEnv} && ${cleanCmd}"
      workingDir "${projectDir}"
    }
  }
}

task configure(type: Exec) {
  environment makeEnvironment
  commandLine shell
  args shellFlag, "${setEnv} && cmake -B ${cmakeBuildDir} ${cmakeFlags}"
  workingDir "${projectDir}"
}

task compile(type: Exec) {
  environment makeEnvironment
  commandLine shell
  args shellFlag, "${setEnv} && cmake --build ${cmakeBuildDir} --config ${cmakeBuildType} -v -j ${numCpus} -t install"
  workingDir "${projectDir}"
  dependsOn 'configure'
}

task strip(type: Exec) {
  environment makeEnvironment
  commandLine shell
  args shellFlag, "${setEnv} && ${stripCmd}"
  workingDir "${projectDir}"
  dependsOn 'compile'
}

task format(type: Exec) {
  commandLine shell
  args shellFlag, "${setEnv} && cmake -P cmake/clang-format.cmake"
  workingDir "${projectDir}"
}

task buildZip(type: Zip) {
  dependsOn strip
  archiveClassifier = artifactClassifier
  from("${buildDir}/distribution") {
    // Don't copy Windows import libraries
    exclude "**/*.lib"
    // Don't copy the test support libraries
    exclude "**/*unit_test_framework*"
    exclude "**/libMlTest.*"
    // Don't copy debug files
    exclude "**/*-debug"
    exclude "**/*.pdb"
    exclude "**/*.dSYM/**"
    // Don't copy core dumps
    exclude "**/core*"
    includeEmptyDirs = false
  }
}

task buildZipSymbols(type: Zip) {
  dependsOn strip
  archiveClassifier = "debug-${artifactClassifier}"
  from("${buildDir}/distribution") {
    // only take debug files
    include "**/*-debug"
    include "**/*.pdb"
    include "**/*.dSYM/**"
    includeEmptyDirs = false
  }
}

task buildUberZip(type: Zip) {
  dependsOn strip

  with buildZip
  // We might also have binaries for other platforms (e.g. if they've been built in Docker)
  def localZips = fileTree("${buildDir}/distributions").matching {
      include "${artifactName}-${project.version}-darwin-*.zip"
      include "${artifactName}-${project.version}-linux-*.zip"
      include "${artifactName}-${project.version}-windows-*.zip"
  }
  for (zipFile in localZips) {
    from(zipTree(zipFile)) {
      duplicatesStrategy 'exclude'
    }
  }
  reproducibleFileOrder = true
}


String artifactGroupPath = project.group.replaceAll("\\.", "/")

task test(type: Exec) {
  environment makeEnvironment
  commandLine shell
  args shellFlag, "${setEnv} && cmake --build ${cmakeBuildDir} --config ${cmakeBuildType} -v -j ${numCpus} -t test"
  workingDir "${projectDir}"
  dependsOn 'strip'
  description = 'Run C++ tests'
}

check {
  dependsOn = ['test']
  description = 'Run all verification tasks'
}

assemble {
  dependsOn = ['buildUberZip', 'buildZip', 'buildZipSymbols']
  description = 'Assemble the C++ part of Machine Learning'
}

build {
  description = 'Assemble and test the C++ part of Machine Learning'
}

/**
 * Gradle 6 gets confused if we try to use its standard dependency management to
 * convert artifacts with a classifier to an artifact for the same project
 * without a classifier.  Therefore this class uses low level functionality to
 * get the platform-specific artifacts.
 */
class DownloadPlatformSpecific extends DefaultTask {

  @Input
  String version = project.version

  @Input
  String artifactGroupPath = project.group.replaceAll("\\.", "/")

  /**
   * Base name for the artifacts
   */
  @Input
  String baseName

  /**
   * Directory to download platform specific zip files into
   */
  @Input
  String downloadDirectory

  /**
   * Directory to extract downloaded zip files into
   */
  @OutputDirectory
  File extractDirectory

  @Input
  List<String> platforms = [ 'darwin-aarch64', 'darwin-x86_64', 'linux-aarch64', 'linux-x86_64', 'windows-x86_64' ]

  DownloadPlatformSpecific() {
    // Always run this task, in case the platform-specific zips have changed
    outputs.upToDateWhen {
      return false
    }
  }

  @TaskAction
  void combine() {
    extractDirectory.deleteDir()
    platforms.each {
      File zipFile = new File(downloadDirectory, "${baseName}-${version}-${it}.zip")
      zipFile.parentFile.mkdirs()
      new URL("https://prelert-artifacts.s3.amazonaws.com/maven/${artifactGroupPath}/${baseName}/${version}/${zipFile.name}").withInputStream { i ->
        zipFile.withOutputStream { o ->
          o << i
        }
      }
      ZipFile zip = new ZipFile(zipFile)
      zip.entries().each {
        File target = new File(extractDirectory, it.name)
        // There can be overlaps between the platform-specific zips, so skip duplicates
        if (target.exists() == false) {
          if (it.isDirectory()) {
            target.mkdirs()
          } else {
            target.parentFile.mkdirs()
            target.withOutputStream { o ->
              o << zip.getInputStream(it)
            }
            target.setLastModified(it.getTime())
          }
        }
      }
      zip.close()
      // Also download the corresponding zip of debug symbols, but there's no need to extract this
      File debugZipFile = new File(downloadDirectory, "${baseName}-${version}-debug-${it}.zip")
      new URL("https://prelert-artifacts.s3.amazonaws.com/maven/${artifactGroupPath}/${baseName}/${version}/${debugZipFile.name}").withInputStream { i ->
        debugZipFile.withOutputStream { o ->
          o << i
        }
      }
    }
  }
}

task downloadPlatformSpecific(type: DownloadPlatformSpecific) {
  description = 'Download and extract previously created platform-specific C++ zips'
  baseName = artifactName
  downloadDirectory = "${buildDir}/distributions"
  extractDirectory = file("${buildDir}/temp")
}

task buildUberZipFromDownloads(type: Zip) {
  description = 'Create an uber zip from combined platform-specific C++ distributions'
  destinationDirectory = file("${buildDir}/distributions")
  from(downloadPlatformSpecific)
  reproducibleFileOrder = true
}

def dependenciesSpec(source) {
  return copySpec {
    from(source) {
      // Don't copy ML libraries
      exclude "**/libMl*"
      // Don't copy ML programs
      exclude "platform/darwin*/controller.app/Contents/MacOS/*"
      exclude "platform/linux*/bin/*"
      exclude "platform/windows*/bin/*.exe"
      // Don't copy resources
      exclude "**/ml-en.dict"
      exclude "**/Info.plist"
      exclude "**/date_time_zonespec.csv"
      // Don't copy licenses
      exclude "**/licenses/**"
      includeEmptyDirs = false
    }
  }
}

task buildDependenciesZip(type: Zip) {
  description = 'Create a zip of dependencies only from combined platform-specific C++ distributions'

  archiveClassifier = 'deps'
  reproducibleFileOrder = true
  // Drop the file timestamps for the dependencies to avoid
  // timestamp diffs between otherwise identical archives
  preserveFileTimestamps = false

  with dependenciesSpec(zipTree(buildUberZip.archiveFile))
}

task buildDependenciesZipFromDownloads(type: Zip) {
  description = 'Create a zip of dependencies only from combined platform-specific C++ distributions from published snapshot'

  archiveClassifier = 'deps'
  reproducibleFileOrder = true
  // Drop the file timestamps for the dependencies to avoid
  // timestamp diffs between otherwise identical archives
  preserveFileTimestamps = false

  with dependenciesSpec(downloadPlatformSpecific)
}


def noDependenciesSpec(source) {
   return copySpec {
    from(source) {
      // Copy ML libraries
      include "**/libMl*"
      // Copy ML programs
      include "platform/darwin*/controller.app/Contents/MacOS/*"
      include "platform/linux*/bin/*"
      include "platform/windows*/bin/*.exe"
      // Copy resources
      include "**/ml-en.dict"
      include "**/Info.plist"
      include "**/date_time_zonespec.csv"
      // Copy licenses
      include "**/licenses/**"
      includeEmptyDirs = false
    }
  }
}

task buildNoDependenciesZip(type: Zip) {
  description = 'Create a zip excluding dependencies from combined platform-specific C++ distributions'

  archiveClassifier = 'nodeps'
  reproducibleFileOrder = true
  // Drop the file timestamps for the dependencies to avoid
  // timestamp diffs between otherwise identical archives
  preserveFileTimestamps = false

  with noDependenciesSpec(zipTree(buildUberZip.archiveFile))
}

task buildNoDependenciesZipFromDownloads(type: Zip) {
  description = 'Create a zip excluding dependencies from combined platform-specific C++ distributions from published snapshot'

  archiveClassifier = 'nodeps'
  reproducibleFileOrder = true
  // Drop the file timestamps for the dependencies to avoid
  // timestamp diffs between otherwise identical archives
  preserveFileTimestamps = false

  with noDependenciesSpec(downloadPlatformSpecific)
}

task buildDependencyReport(type: Exec) {
  outputs.file("${buildDir}/distributions/dependencies-${version}.csv")
  environment makeEnvironment
  commandLine shell
  args shellFlag, isWindows ? "${setEnv} && cmake -D OUTPUT_FILE=\"${outputs.files.singleFile}\" -P 3rd_party/dependency_report.cmake"
                            : "${setEnv} && 3rd_party/dependency_report.sh --csv \"${outputs.files.singleFile}\""
  workingDir "${projectDir}"
  description = 'Create a CSV report on 3rd party dependencies we redistribute'
}

// This intentionally doesn't depend on assemble.
// This gives us the flexibility to build in different
// ways and still use the same upload code.
task upload(type: UploadS3Task) {
  bucket 'prelert-artifacts'
  // Only upload the platform-specific artifacts in this task
  def zipFileDir = fileTree("${buildDir}/distributions").matching {
    include "*-aarch64.zip", "*-x86_64.zip"
  }
  for (zipFile in zipFileDir) {
    upload zipFile, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${zipFile.name}"
  }
  description = 'Upload C++ zips to S3 Bucket'
}

task uploadAll(type: UploadS3Task) {
  bucket 'prelert-artifacts'
  // Upload ALL artifacts (including the dependency report) in this task
  def fileDir = fileTree("${buildDir}/distributions").matching {
    include "ml-cpp-${project.version}*.zip", "dependencies-${version}.csv"
  }
  for (file in fileDir) {
    // TODO: swap back when Jenkins is switched off
    upload file, "maven/${artifactGroupPath}/${artifactName}/BuildKite/${file.name}"
    // upload file, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${file.name}"
  }
  description = 'Upload all C++ artifacts to S3 Bucket'
}

task uberUpload(type: UploadS3Task, dependsOn: [buildUberZipFromDownloads,
                                                buildDependenciesZipFromDownloads,
                                                buildNoDependenciesZipFromDownloads,
                                                buildDependencyReport]) {
  bucket 'prelert-artifacts'
  upload buildUberZipFromDownloads.outputs.files.singleFile, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${buildUberZipFromDownloads.outputs.files.singleFile.name}"
  upload buildDependenciesZipFromDownloads.outputs.files.singleFile, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${buildDependenciesZipFromDownloads.outputs.files.singleFile.name}"
  upload buildNoDependenciesZipFromDownloads.outputs.files.singleFile, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${buildNoDependenciesZipFromDownloads.outputs.files.singleFile.name}"
  upload buildDependencyReport.outputs.files.singleFile, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${buildDependencyReport.outputs.files.singleFile.name}"
  description = 'Upload C++ uber zip and dependency report to S3 Bucket'
}

wrapper {
  distributionType = 'ALL'
  doLast {
    final DistributionLocator locator = new DistributionLocator()
    final GradleVersion version = GradleVersion.version(wrapper.gradleVersion)
    final URI distributionUri = locator.getDistributionFor(version, wrapper.distributionType.name().toLowerCase(Locale.ENGLISH))
    final URI sha256Uri = new URI(distributionUri.toString() + ".sha256")
    final String sha256Sum = new String(sha256Uri.toURL().bytes)
    wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n"
    println "Added checksum to wrapper properties"
  }
}

artifacts {
  'default' buildDependenciesZip
  'default' buildNoDependenciesZip
}