build_llvm.py

#!/usr/bin/env python3
#
# Copyright (C) 2019 Intel Corporation.  All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#

import argparse
import os
import pathlib
import requests
import shlex
import shutil
import subprocess
import sysconfig
import sys


def clone_llvm(dst_dir, llvm_repo, llvm_branch):
    """
    any error will raise CallProcessError
    """
    llvm_dir = dst_dir.joinpath("llvm").resolve()

    if not llvm_dir.exists():
        GIT_CLONE_CMD = f"git clone --depth 1 --branch {llvm_branch} {llvm_repo} llvm"
        print(GIT_CLONE_CMD)
        subprocess.check_output(shlex.split(GIT_CLONE_CMD), cwd=dst_dir)

    return llvm_dir


def query_llvm_version(llvm_info):
    github_token = os.environ['GH_TOKEN']
    owner_project = llvm_info['repo'].replace("https://github.com/", "").replace(".git", "")
    url = f"https://api.github.com/repos/{owner_project}/commits/{llvm_info['branch']}"
    headers = {
        'Authorization': f"Bearer {github_token}"
    }

    try:
        response = requests.request("GET", url, headers=headers, data={})
        response.raise_for_status()
    except requests.exceptions.HTTPError as error:
        print (error) # for debugging purpose
        return None

    response = response.json()
    return response['sha']


def build_llvm(llvm_dir, platform, backends, projects, use_clang=False, extra_flags=''):
    LLVM_COMPILE_OPTIONS = [
        '-DCMAKE_BUILD_TYPE:STRING="Release"',
        "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
        "-DLLVM_APPEND_VC_REV:BOOL=ON",
        "-DLLVM_BUILD_EXAMPLES:BOOL=OFF",
        "-DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF",
        "-DLLVM_ENABLE_BINDINGS:BOOL=OFF",
        "-DLLVM_ENABLE_IDE:BOOL=OFF",
        "-DLLVM_ENABLE_LIBEDIT=OFF",
        "-DLLVM_ENABLE_TERMINFO:BOOL=OFF",
        "-DLLVM_ENABLE_ZLIB:BOOL=ON",
        "-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF",
        "-DLLVM_INCLUDE_DOCS:BOOL=OFF",
        "-DLLVM_INCLUDE_EXAMPLES:BOOL=OFF",
        "-DLLVM_INCLUDE_UTILS:BOOL=OFF",
        "-DLLVM_INCLUDE_TESTS:BOOL=OFF",
        "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON",
    ]

    # ccache is not available on Windows
    if not "windows" == platform:
        LLVM_COMPILE_OPTIONS.append("-DLLVM_CCACHE_BUILD:BOOL=ON")
    # perf support is available on Linux only
    if "linux" == platform:
        LLVM_COMPILE_OPTIONS.append("-DLLVM_USE_PERF:BOOL=ON")

    # use clang/clang++/lld. but macos doesn't support lld
    if not sys.platform.startswith("darwin") and use_clang:
        if shutil.which("clang") and shutil.which("clang++") and shutil.which("lld"):
            os.environ["CC"] = "clang"
            os.environ["CXX"] = "clang++"
            LLVM_COMPILE_OPTIONS.append('-DLLVM_USE_LINKER:STRING="lld"')
            print("Use the clang toolchain")
        else:
            print("Can not find clang, clang++ and lld, keep using the gcc toolchain")
    else:
        print("Use the gcc toolchain")

    LLVM_EXTRA_COMPILE_OPTIONS = {
        "arc": [
            '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="ARC"',
            "-DLLVM_ENABLE_LIBICUUC:BOOL=OFF",
            "-DLLVM_ENABLE_LIBICUDATA:BOOL=OFF",
        ],
        "xtensa": [
            '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="Xtensa"',
        ],
        "windows": [
            "-DCMAKE_INSTALL_PREFIX=LLVM-install",
        ],
        "default": [],
    }

    experimental_backends = ["ARC", "Xtensa"]
    normal_backends = [s for s in backends if s not in experimental_backends]

    LLVM_TARGETS_TO_BUILD = [
        '-DLLVM_TARGETS_TO_BUILD:STRING="' + ";".join(normal_backends) + '"'
        if normal_backends
        else '-DLLVM_TARGETS_TO_BUILD:STRING="AArch64;ARM;Mips;RISCV;X86"'
    ]

    # if not on ARC platform, but want to add expeirmental backend ARC as target
    if platform != "ARC" and "ARC" in backends: 
        LLVM_TARGETS_TO_BUILD.extend(
            LLVM_EXTRA_COMPILE_OPTIONS["arc"]
        )

    if platform != "Xtensa" and "Xtensa" in backends:
        print(
            "Currently it's not supported to build Xtensa backend on non-Xtensa platform"
        )
        return None

    LLVM_PROJECTS_TO_BUILD = [
        '-DLLVM_ENABLE_PROJECTS:STRING="' + ";".join(projects) + '"' if projects else ""
    ]

    # lldb project requires libxml2
    LLVM_LIBXML2_OPTION = [
        "-DLLVM_ENABLE_LIBXML2:BOOL=" + ("ON" if "lldb" in projects else "OFF")
    ]

    # enabling LLVM_INCLUDE_TOOLS will increase ~300M to the final package
    LLVM_INCLUDE_TOOLS_OPTION = [
        "-DLLVM_INCLUDE_TOOLS:BOOL=ON" if projects else "-DLLVM_INCLUDE_TOOLS:BOOL=OFF"
    ]

    if not llvm_dir.exists():
        raise Exception(f"{llvm_dir} doesn't exist")

    build_dir = llvm_dir.joinpath("build").resolve()
    build_dir.mkdir(exist_ok=True)

    lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve()
    if lib_llvm_core_library.exists():
        print(
            f"It has already been fully compiled. If want to a re-build, please remove {build_dir} manually and try again"
        )
        return None

    compile_options = " ".join(
        LLVM_COMPILE_OPTIONS
        + LLVM_LIBXML2_OPTION
        + LLVM_EXTRA_COMPILE_OPTIONS.get(
            platform, LLVM_EXTRA_COMPILE_OPTIONS["default"]
        )
        + LLVM_TARGETS_TO_BUILD
        + LLVM_PROJECTS_TO_BUILD
        + LLVM_INCLUDE_TOOLS_OPTION
    )

    CONFIG_CMD = f"cmake {compile_options} {extra_flags} ../llvm"
    if "windows" == platform:
        if "mingw" in sysconfig.get_platform().lower():
            CONFIG_CMD += " -G'Unix Makefiles'"
        else:
            CONFIG_CMD += " -A x64"
    else:
        CONFIG_CMD += " -G'Ninja'"
    print(f"Config command: {CONFIG_CMD}")
    subprocess.check_call(shlex.split(CONFIG_CMD), cwd=build_dir)

    BUILD_CMD = "cmake --build . --target package" + (
        " --config Release" if "windows" == platform else ""
    )
    if "windows" == platform:
        BUILD_CMD += " --parallel " + str(os.cpu_count())
    print(f"Build command: {BUILD_CMD}")
    subprocess.check_call(shlex.split(BUILD_CMD), cwd=build_dir)

    return build_dir


def repackage_llvm(llvm_dir):
    build_dir = llvm_dir.joinpath("./build").resolve()

    packs = [f for f in build_dir.glob("LLVM-*.tar.gz")]
    if len(packs) > 1:
        raise Exception("Find more than one LLVM-*.tar.gz")

    if not packs:
        raise Exception("Didn't find any LLVM-* package")
        return

    llvm_package = packs[0].name
    # mv build/LLVM-*.gz .
    shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir))
    # rm -r build
    shutil.rmtree(str(build_dir))
    # mkdir build
    build_dir.mkdir()
    # tar xf ./LLVM-*.tar.gz --strip-components=1 --directory=build
    CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}"
    subprocess.check_call(shlex.split(CMD), cwd=llvm_dir)
    # rm ./LLVM-1*.gz
    os.remove(llvm_dir.joinpath(llvm_package).resolve())

def repackage_llvm_windows(llvm_dir):
    build_dir = llvm_dir.joinpath("./build").resolve()

    packs_path = [f for f in build_dir.glob("./_CPack_Packages/win64/NSIS/LLVM-*-win64")]
    if len(packs_path) > 1:
        raise Exception("Find more than one LLVM-* package")

    if not packs_path:
        raise Exception("Didn't find any LLVM-* package")
        return
    
    llvm_package_path = f"_CPack_Packages/win64/NSIS/{packs_path[0].name}"
    windows_package_dir = build_dir.joinpath(llvm_package_path).resolve()

    # mv package dir outside of build 
    shutil.move(str(windows_package_dir), str(llvm_dir))
    # rm -r build
    shutil.rmtree(str(build_dir))
    # mkdir build
    build_dir.mkdir()
    # move back all the subdiretories under cpack directory(bin/include/lib) to build dir
    moved_package_dir = llvm_dir.joinpath(packs_path[0].name)
    for sub_dir in moved_package_dir.iterdir():
        shutil.move(str(sub_dir), str(build_dir))
    moved_package_dir.rmdir()  

def main():
    parser = argparse.ArgumentParser(description="build necessary LLVM libraries")
    parser.add_argument(
        "--platform",
        type=str,
        choices=["android", "arc", "darwin", "linux", "windows", "xtensa"],
        help="identify current platform",
    )
    parser.add_argument(
        "--arch",
        nargs="+",
        type=str,
        choices=[
            "AArch64",
            "ARC",
            "ARM",
            "Mips",
            "RISCV",
            "WebAssembly",
            "X86",
            "Xtensa",
        ],
        default=[],
        help="identify LLVM supported backends, separate by space, like '--arch ARM Mips X86'",
    )
    parser.add_argument(
        "--project",
        nargs="+",
        type=str,
        default="",
        choices=["clang", "lldb"],
        help="identify extra LLVM projects, separate by space, like '--project clang lldb'",
    )
    parser.add_argument(
        "--llvm-ver",
        action="store_true",
        help="return the version info of generated llvm libraries",
    )
    parser.add_argument(
        "--use-clang",
        action="store_true",
        help="use clang instead of gcc",
    )
    parser.add_argument(
        "--extra-cmake-flags",
        type=str,
        default="",
        help="custom extra cmake flags",
    )
    options = parser.parse_args()

    # if the "platform" is not identified in the command line option,
    # detect it
    if not options.platform:
        if sys.platform.startswith("win32") or sys.platform.startswith("msys"):
            platform = "windows"
        elif sys.platform.startswith("darwin"):
            platform = "darwin"
        else:
            platform = "linux"
    else:
        platform = options.platform

    llvm_repo_and_branch = {
        "arc": {
            "repo": "https://github.com/llvm/llvm-project.git",
            "repo_ssh": "git@github.com:llvm/llvm-project.git",
            "branch": "release/15.x",
        },
        "xtensa": {
            "repo": "https://github.com/espressif/llvm-project.git",
            "repo_ssh": "git@github.com:espressif/llvm-project.git",
            "branch": "xtensa_release_17.0.1",
        },
        "default": {
            "repo": "https://github.com/llvm/llvm-project.git",
            "repo_ssh": "git@github.com:llvm/llvm-project.git",
            "branch": "release/15.x",
        },
    }

    # retrieve the real file
    current_file = pathlib.Path(__file__)
    if current_file.is_symlink():
        current_file = pathlib.Path(os.readlink(current_file))

    current_dir = current_file.parent.resolve()
    deps_dir = current_dir.joinpath("../core/deps").resolve()

    try:
        llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])

        if options.llvm_ver:
            commit_hash = query_llvm_version(llvm_info)
            print(commit_hash)
            return commit_hash is not None

        repo_addr = llvm_info["repo"]
        if os.environ.get('USE_GIT_SSH') == "true":
            repo_addr = llvm_info["repo_ssh"]
        else:
            print("To use ssh for git clone, run: export USE_GIT_SSH=true")

        llvm_dir = clone_llvm(deps_dir, repo_addr, llvm_info["branch"])
        if (
            build_llvm(
                llvm_dir, platform, options.arch, options.project, options.use_clang,
                options.extra_cmake_flags
            )
            is not None
        ):
            # TODO: repackage process may change in the future, this work for LLVM 15.x
            if "windows" == platform:
                repackage_llvm_windows(llvm_dir)
            else:
                repackage_llvm(llvm_dir)

        return True
    except subprocess.CalledProcessError:
        return False


if __name__ == "__main__":
    sys.exit(0 if main() else 1)