generate_compdb.py

#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This file is a combination of tools/clang/scripts/generate_compdb.py and
# tools/clang/pylib/clang/compile_db.py from the Chromium source code.
# They are modified to use the ninja executable in PATH, rather than finding
# the binary in the Chromium directory structure.
#
# Delete when tools/clang is updated to include this commit:
# https://chromium.googlesource.com/chromium/src/tools/clang.git/+/d324a17c34dba948e42565378bcdfdac919e62c2

"""
Helper for generating compile DBs for clang tooling. On non-Windows platforms,
this is pretty straightforward. On Windows, the tool does a bit of extra work to
integrate the content of response files, force clang tooling to run in clang-cl
mode, etc.
"""

import argparse
import json
import os
import re
import sys
import subprocess


_RSP_RE = re.compile(r' (@(.+?\.rsp)) ')
_CMD_LINE_RE = re.compile(
    r'^(?P<gomacc>.*gomacc(\.exe)?"?\s+)?(?P<clang>\S*clang\S*)\s+(?P<args>.*)$'
)
_debugging = False


def _IsTargettingWindows(target_os):
    if target_os is not None:
        # Available choices are based on: gn help target_os
        assert target_os in [
            'android', 'chromeos', 'ios', 'linux', 'nacl', 'mac', 'win'
        ]
        return target_os == 'win'
    return sys.platform == 'win32'


def _ProcessCommand(command, target_os):
    """Removes gomacc(.exe). On Windows inserts --driver-mode=cl as the first arg.

    Note that we deliberately don't use shlex.split here, because it doesn't work
    predictably for Windows commands (specifically, it doesn't parse args the same
    way that Clang does on Windows).

    Instead, we just use a regex, with the simplifying assumption that the path to
    clang-cl.exe contains no spaces.
    """
    # If the driver mode is not already set then define it. Driver mode is
    # automatically included in the compile db by clang starting with release
    # 9.0.0.
    driver_mode = ''
    # Only specify for Windows. Other platforms do fine without it.
    if _IsTargettingWindows(target_os) and '--driver-mode' not in command:
        driver_mode = '--driver-mode=cl'

    match = _CMD_LINE_RE.search(command)
    if match:
        match_dict = match.groupdict()
        command = ' '.join(
            [match_dict['clang'], driver_mode, match_dict['args']])
    elif _debugging:
        print('Compile command didn\'t match expected regex!')
        print('Command:', command)
        print('Regex:', _CMD_LINE_RE.pattern)

    # Remove some blocklisted arguments. These are VisualStudio specific arguments
    # not recognized or used by clangd. They only suppress or activate graphical
    # output anyway.
    blocklisted_arguments = ['/nologo', '/showIncludes']
    command_parts = filter(lambda arg: arg not in blocklisted_arguments,
                           command.split())

    return " ".join(command_parts)


def _ProcessEntry(entry, target_os):
    """Transforms one entry in a Windows compile db to be clang-tool friendly."""
    entry['command'] = _ProcessCommand(entry['command'], target_os)

    # Expand the contents of the response file, if any.
    # http://llvm.org/bugs/show_bug.cgi?id=21634
    try:
        match = _RSP_RE.search(entry['command'])
        if match:
            rsp_path = os.path.join(entry['directory'], match.group(2))
            rsp_contents = open(rsp_path).read()
            entry['command'] = ''.join([
                entry['command'][:match.start(1)],
                rsp_contents,
                entry['command'][match.end(1):]])
    except IOError:
        if _debugging:
            print('Couldn\'t read response file for %s' % entry['file'])

    return entry


def ProcessCompileDatabaseIfNeeded(compile_db, target_os=None):
    """Make the compile db generated by ninja on Windows more clang-tool friendly.

    Args:
      compile_db: The compile database parsed as a Python dictionary.

    Returns:
      A postprocessed compile db that clang tooling can use.
    """
    compile_db = [_ProcessEntry(e, target_os) for e in compile_db]

    if not _IsTargettingWindows(target_os):
        return compile_db

    if _debugging:
        print('Read in %d entries from the compile db' % len(compile_db))
    original_length = len(compile_db)

    # Filter out NaCl stuff. The clang tooling chokes on them.
    # TODO(dcheng): This doesn't appear to do anything anymore, remove?
    compile_db = [e for e in compile_db if '_nacl.cc.pdb' not in e['command']
                  and '_nacl_win64.cc.pdb' not in e['command']]
    if _debugging:
        print('Filtered out %d entries...' %
              (original_length - len(compile_db)))

    # TODO(dcheng): Also filter out multiple commands for the same file. Not sure
    # how that happens, but apparently it's an issue on Windows.
    return compile_db


def GetNinjaExecutable():
    return 'ninja.exe' if sys.platform == 'win32' else 'ninja'


# FIXME: This really should be a build target, rather than generated at runtime.
def GenerateWithNinja(path, targets=[]):
    """Generates a compile database using ninja.

    Args:
      path: The build directory to generate a compile database for.
      targets: Additional targets to pass to ninja.

    Returns:
      List of the contents of the compile database.
    """
    # TODO(dcheng): Ensure that clang is enabled somehow.

    # First, generate the compile database.
    json_compile_db = subprocess.check_output(
        [GetNinjaExecutable(), '-C', path] + targets +
        ['-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx'])
    return json.loads(json_compile_db)


def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-p',
        required=True,
        help='Path to build directory')
    parser.add_argument(
        'targets',
        nargs='*',
        help='Additional targets to pass to ninja')
    parser.add_argument(
        '--target_os',
        choices=['android', 'chromeos', 'ios', 'linux', 'nacl', 'mac', 'win'],
        help='Target OS - see `gn help target_os`. Set to "win" when ' +
        'cross-compiling Windows from Linux or another host')
    parser.add_argument(
        '-o',
        help='File to write the compilation database to. Defaults to stdout')

    args = parser.parse_args()

    compdb_text = json.dumps(ProcessCompileDatabaseIfNeeded(
        GenerateWithNinja(args.p, args.targets), args.target_os),
        indent=2)
    if args.o is None:
        print(compdb_text)
    else:
        with open(args.o, 'w') as f:
            f.write(compdb_text)


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))