From d46d158e35954357d3830edae6464f12273798ed Mon Sep 17 00:00:00 2001 From: Yohei Yukawa Date: Thu, 14 Nov 2024 00:25:04 +0900 Subject: [PATCH] WIP --- .gitignore | 1 + src/build_tools/build_qt.py | 285 +++++++++++++++++++++++++++++++++++- src/build_tools/vs_util.py | 32 +++- 3 files changed, 303 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 2a3bf0a988..30098582c1 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ MODULE.bazel.lock # third_party dirs and cache dir checked out by update_deps.py /src/third_party/ninja/ /src/third_party/qt/ +/src/third_party/qt_host/ /src/third_party/qt_src/ /src/third_party/wix/ /src/third_party_cache/ diff --git a/src/build_tools/build_qt.py b/src/build_tools/build_qt.py index 6430e283f1..06d4db6403 100755 --- a/src/build_tools/build_qt.py +++ b/src/build_tools/build_qt.py @@ -52,6 +52,7 @@ import dataclasses import functools import os +import platform import pathlib import shutil import subprocess @@ -67,6 +68,7 @@ ABS_MOZC_SRC_DIR = ABS_SCRIPT_PATH.parents[1] ABS_QT_SRC_DIR = ABS_MOZC_SRC_DIR.joinpath('third_party', 'qt_src') ABS_QT_DEST_DIR = ABS_MOZC_SRC_DIR.joinpath('third_party', 'qt') +ABS_QT_HOST_DIR = ABS_MOZC_SRC_DIR.joinpath('third_party', 'qt_host') # The archive filename should be consistent with update_deps.py. ABS_QT6_ARCHIVE_PATH = ABS_MOZC_SRC_DIR.joinpath( 'third_party_cache', 'qtbase-everywhere-src-6.8.0.tar.xz') @@ -285,7 +287,9 @@ def get_qt_version(args: argparse.Namespace) -> QtVersion: ) -def make_configure_options(args: argparse.Namespace) -> list[str]: +def make_host_configure_options( + args: argparse.Namespace, +) -> list[str]: """Makes necessary configure options based on args. Args: @@ -306,6 +310,7 @@ def make_configure_options(args: argparse.Namespace) -> list[str]: qt_configure_options = ['-opensource', '-c++std', 'c++20', '-silent', + '-no-accessibility', '-no-cups', '-no-dbus', '-no-feature-androiddeployqt', @@ -352,6 +357,7 @@ def make_configure_options(args: argparse.Namespace) -> list[str]: '-no-feature-windeployqt', '-no-feature-wizard', '-no-feature-xml', + '-no-gui', '-no-icu', '-no-opengl', '-no-sql-db2', @@ -361,8 +367,121 @@ def make_configure_options(args: argparse.Namespace) -> list[str]: '-no-sql-odbc', '-no-sql-psql', '-no-sql-sqlite', + '-no-widgets', '-nomake', 'examples', '-nomake', 'tests', + '-nomake', 'benchmarks', + '-nomake', 'manual-tests', + '-nomake', 'minimal-static-tests', + '-make', 'tools', + '-release', + '-static', + ] + + if is_mac(): + qt_configure_options += [ + '-platform', 'macx-clang', + '-qt-libpng', + '-qt-pcre', + ] + elif is_windows(): + qt_configure_options += ['-no-freetype', + '-no-harfbuzz', + '-platform', 'win32-msvc'] + if args.confirm_license: + qt_configure_options += ['-confirm-license'] + + qt_src_dir = pathlib.Path(args.qt_src_dir).resolve() + qt_host_dir = pathlib.Path(args.qt_host_dir).resolve() + if qt_src_dir != qt_host_dir: + qt_configure_options += ['-prefix', str(qt_host_dir)] + + return qt_configure_options + + +def make_configure_options( + args: argparse.Namespace, +) -> list[str]: + """Makes necessary configure options based on args. + + Args: + args: build options to be used to customize configure options of Qt. + + Returns: + A list of configure options to be passed to configure of Qt. + Raises: + ValueError: When Qt major version is not 6. + ValueError: When --macos_cpus=arm64 is set on Intel64 mac. + """ + + qt_version = get_qt_version(args) + + if qt_version.major != 6: + raise ValueError(f'Only Qt6 is supported but specified {qt_version}.') + + qt_configure_options = ['-opensource', + '-c++std', 'c++20', + '-silent', + '-no-cups', + '-no-dbus', + '-no-feature-androiddeployqt', + '-no-feature-animation', + '-no-feature-calendarwidget', + '-no-feature-completer', + '-no-feature-concatenatetablesproxymodel', + '-no-feature-concurrent', + '-no-feature-dial', + '-no-feature-effects', + '-no-feature-fontcombobox', + '-no-feature-fontdialog', + '-no-feature-identityproxymodel', + '-no-feature-image_heuristic_mask', + '-no-feature-imageformatplugin', + '-no-feature-islamiccivilcalendar', + '-no-feature-itemmodeltester', + '-no-feature-jalalicalendar', + '-no-feature-macdeployqt', + '-no-feature-mdiarea', + '-no-feature-mimetype', + '-no-feature-movie', + '-no-feature-network', + '-no-feature-poll-exit-on-error', + '-no-feature-qmake', + '-no-feature-sha3-fast', + '-no-feature-sharedmemory', + '-no-feature-socks5', + '-no-feature-splashscreen', + '-no-feature-sql', + '-no-feature-sqlmodel', + '-no-feature-sspi', + '-no-feature-stringlistmodel', + '-no-feature-tabletevent', + '-no-feature-testlib', + '-no-feature-textbrowser', + '-no-feature-textmarkdownreader', + '-no-feature-textmarkdownwriter', + '-no-feature-textodfwriter', + '-no-feature-timezone', + '-no-feature-topleveldomain', + '-no-feature-undoview', + '-no-feature-whatsthis', + '-no-feature-windeployqt', + '-no-feature-wizard', + '-no-feature-xml', + '-no-icu', + '-no-opengl', + '-no-sql-db2', + '-no-sql-ibase', + '-no-sql-mysql', + '-no-sql-oci', + '-no-sql-odbc', + '-no-sql-psql', + '-no-sql-sqlite', + '-nomake', 'examples', + '-nomake', 'tests', + '-nomake', 'benchmarks', + '-nomake', 'manual-tests', + '-nomake', 'minimal-static-tests', ] cmake_options = [] @@ -394,11 +513,13 @@ def make_configure_options(args: argparse.Namespace) -> list[str]: elif is_windows(): qt_configure_options += ['-force-debug-info', - '-intelcet', '-ltcg', # Note: ignored in debug build '-no-freetype', '-no-harfbuzz', '-platform', 'win32-msvc'] + if args.windows_cpu in ['x64', 'amd64']: + qt_configure_options += ['-intelcet'] + if args.confirm_license: qt_configure_options += ['-confirm-license'] @@ -414,6 +535,7 @@ def make_configure_options(args: argparse.Namespace) -> list[str]: qt_src_dir = pathlib.Path(args.qt_src_dir).resolve() qt_dest_dir = pathlib.Path(args.qt_dest_dir).resolve() + if qt_src_dir != qt_dest_dir: qt_configure_options += ['-prefix', str(qt_dest_dir)] @@ -436,6 +558,8 @@ def parse_args() -> argparse.Namespace: default=str(ABS_QT6_ARCHIVE_PATH)) parser.add_argument('--qt_dest_dir', help='qt dest directory', type=str, default=str(ABS_QT_DEST_DIR)) + parser.add_argument('--qt_host_dir', help='qt host tools directory', type=str, + default=str(ABS_QT_HOST_DIR)) parser.add_argument('--confirm_license', help='set to accept Qt OSS license', action='store_true', default=False) @@ -443,6 +567,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument('--ninja_dir', help='Directory of ninja executable', type=str, default=None) if is_windows(): + parser.add_argument('--windows_cpu', + help='"x64" or "arm64"', type=str, default="x64") parser.add_argument('--vcvarsall_path', help='Path of vcvarsall.bat', type=str, default=None) elif is_mac(): @@ -476,6 +602,51 @@ def get_ninja_dir(args: argparse.Namespace) -> Union[pathlib.Path, None]: return None +def build_host_on_mac(args: argparse.Namespace) -> None: + """Build Qt from the source code on Mac. + + + Args: + args: build options to be used to customize configure options of Qt. + Raises: + FileNotFoundError: when any required file is not found. + """ + extract_qt_src(args) + + qt_src_dir = pathlib.Path(args.qt_src_dir).resolve() + qt_host_dir = pathlib.Path(args.qt_host_dir).resolve() + + if not (args.dryrun or qt_src_dir.exists()): + raise FileNotFoundError('Could not find qt_src_dir=%s' % qt_src_dir) + + env = dict(os.environ) + + # Use locally checked out ninja.exe if exists. + ninja_dir = get_ninja_dir(args) + if ninja_dir: + env['PATH'] = str(ninja_dir) + os.pathsep + env['PATH'] + + configure_cmds = ['./configure'] + make_host_configure_options(args) + cmake = str(shutil.which('cmake', path=env['PATH'])) + build_cmds = [cmake, '--build', '.', '--parallel'] + install_cmds = [cmake, '--install', '.'] + + exec_command(configure_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) + exec_command(build_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) + + if qt_src_dir == qt_host_dir: + # No need to run 'install' command. + return + + if qt_host_dir.exists(): + if args.dryrun: + print(f'dryrun: delete {qt_host_dir}') + else: + shutil.rmtree(qt_host_dir) + + exec_command(install_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) + + def build_on_mac(args: argparse.Namespace) -> None: """Build Qt from the source code on Mac. @@ -485,10 +656,13 @@ def build_on_mac(args: argparse.Namespace) -> None: Raises: FileNotFoundError: when any required file is not found. """ + extract_qt_src(args) + qt_src_dir = pathlib.Path(args.qt_src_dir).resolve() qt_dest_dir = pathlib.Path(args.qt_dest_dir).resolve() + qt_host_dir = pathlib.Path(args.qt_host_dir).resolve() - if not qt_src_dir.exists(): + if not (args.dryrun or qt_src_dir.exists()): raise FileNotFoundError('Could not find qt_src_dir=%s' % qt_src_dir) env = dict(os.environ) @@ -498,6 +672,8 @@ def build_on_mac(args: argparse.Namespace) -> None: if ninja_dir: env['PATH'] = str(ninja_dir) + os.pathsep + env['PATH'] + env['QT_HOST_PATH'] = str(qt_host_dir) + configure_cmds = ['./configure'] + make_configure_options(args) cmake = str(shutil.which('cmake', path=env['PATH'])) build_cmds = [cmake, '--build', '.', '--parallel'] @@ -518,6 +694,19 @@ def build_on_mac(args: argparse.Namespace) -> None: exec_command(install_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) + for tool in ['moc', 'rcc', 'uic']: + src = qt_host_dir.joinpath('libexec').joinpath(tool) + dest = qt_dest_dir.joinpath('libexec').joinpath(tool) + if args.dryrun: + print(f'dryrun: copy {src} => {dest}') + else: + shutil.copy2(src=src, dst=dest) + + if args.dryrun: + print(f'dryrun: shutil.rmtree({qt_host_dir})') + else: + shutil.rmtree(qt_host_dir) + def exec_command(command: list[str], cwd: Union[str, pathlib.Path], env: dict[str, str], dryrun: bool = False) -> None: @@ -538,6 +727,62 @@ def exec_command(command: list[str], cwd: Union[str, pathlib.Path], subprocess.run(command, shell=False, check=True, cwd=cwd, env=env) +def build_host_on_windows(args: argparse.Namespace) -> None: + """Build Qt from the source code on Windows. + + Args: + args: build options to be used to customize configure options of Qt. + + Raises: + FileNotFoundError: when any required file is not found. + """ + extract_qt_src(args) + + qt_src_dir = pathlib.Path(args.qt_src_dir).resolve() + qt_host_dir = pathlib.Path(args.qt_host_dir).resolve() + + if not (args.dryrun or qt_src_dir.exists()): + raise FileNotFoundError('Could not find qt_src_dir=%s' % qt_src_dir) + + arch = { + 'amd64': 'x64', + 'arm64': 'arm64', + }[platform.uname().machine.lower()] + env = get_vs_env_vars(arch, args.vcvarsall_path) + + # Use locally checked out ninja.exe if exists. + ninja_dir = get_ninja_dir(args) + if ninja_dir: + env['PATH'] = str(ninja_dir) + os.pathsep + env['PATH'] + + # Add qt_src_dir to 'PATH'. + # https://doc.qt.io/qt-6/windows-building.html#step-3-set-the-environment-variables + env['PATH'] = str(qt_src_dir) + os.pathsep + env['PATH'] + + cmd = str(shutil.which('cmd.exe', path=env['PATH'])) + + configure_cmds = [cmd, '/C', 'configure.bat'] + make_host_configure_options(args) + exec_command(configure_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) + + cmake = str(shutil.which('cmake.exe', path=env['PATH'])) + build_cmds = [cmake, '--build', '.', '--parallel'] + install_cmds = [cmake, '--install', '.'] + + exec_command(build_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) + + if qt_src_dir == qt_host_dir: + # No need to run 'install' command. + return + + if qt_host_dir.exists(): + if args.dryrun: + print(f'dryrun: shutil.rmtree({qt_host_dir})') + else: + shutil.rmtree(qt_host_dir) + + exec_command(install_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) + + def build_on_windows(args: argparse.Namespace) -> None: """Build Qt from the source code on Windows. @@ -547,13 +792,22 @@ def build_on_windows(args: argparse.Namespace) -> None: Raises: FileNotFoundError: when any required file is not found. """ + extract_qt_src(args) + qt_src_dir = pathlib.Path(args.qt_src_dir).resolve() qt_dest_dir = pathlib.Path(args.qt_dest_dir).resolve() + qt_host_dir = pathlib.Path(args.qt_host_dir).resolve() - if not qt_src_dir.exists(): + if not (args.dryrun or qt_src_dir.exists()): raise FileNotFoundError('Could not find qt_src_dir=%s' % qt_src_dir) - env = get_vs_env_vars('x64', args.vcvarsall_path) + host_arch = { + 'amd64': 'x64', + 'arm64': 'arm64', + }[platform.uname().machine.lower()] + target_arch = args.windows_cpu + arch = host_arch if host_arch == target_arch else f"{host_arch}_{target_arch}" + env = get_vs_env_vars(arch, args.vcvarsall_path) # Use locally checked out ninja.exe if exists. ninja_dir = get_ninja_dir(args) @@ -565,7 +819,11 @@ def build_on_windows(args: argparse.Namespace) -> None: env['PATH'] = str(qt_src_dir) + os.pathsep + env['PATH'] cmd = str(shutil.which('cmd.exe', path=env['PATH'])) + + env['QT_HOST_PATH'] = str(qt_host_dir) + configure_cmds = [cmd, '/C', 'configure.bat'] + make_configure_options(args) + exec_command(configure_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) cmake = str(shutil.which('cmake.exe', path=env['PATH'])) @@ -592,6 +850,19 @@ def build_on_windows(args: argparse.Namespace) -> None: install_cmds += ['--config', 'debug'] exec_command(install_cmds, cwd=qt_src_dir, env=env, dryrun=args.dryrun) + for bool in ['moc.exe', 'rcc.exe', 'uic.exe']: + src = qt_host_dir.joinpath('bin').joinpath(bool) + dest = qt_dest_dir.joinpath('bin').joinpath(bool) + if args.dryrun: + print(f'dryrun: copy {src} => {dest}') + else: + shutil.copy2(src=src, dst=dest) + + if args.dryrun: + print(f'dryrun: shutil.rmtree({qt_host_dir})') + else: + shutil.rmtree(qt_host_dir) + def extract_qt_src(args: argparse.Namespace) -> None: """Extract Qt src from the archive. @@ -631,11 +902,11 @@ def main(): print('neither --release nor --debug is specified.') sys.exit(1) - extract_qt_src(args) - if is_mac(): + build_host_on_mac(args) build_on_mac(args) elif is_windows(): + build_host_on_windows(args) build_on_windows(args) diff --git a/src/build_tools/vs_util.py b/src/build_tools/vs_util.py index 2226b95d90..89df7538e8 100755 --- a/src/build_tools/vs_util.py +++ b/src/build_tools/vs_util.py @@ -37,10 +37,14 @@ from typing import Union -def get_vcvarsall(path_hint: Union[str, None] = None) -> pathlib.Path: +def get_vcvarsall( + arch: str, + path_hint: Union[str, None] = None +) -> pathlib.Path: """Returns the path of 'vcvarsall.bat'. Args: + arch: host/target architecture path_hint: optional path to vcvarsall.bat Returns: @@ -78,12 +82,17 @@ def get_vcvarsall(path_hint: Union[str, None] = None) -> pathlib.Path: 'Microsoft.VisualStudio.Product.Professional', 'Microsoft.VisualStudio.Product.Community', 'Microsoft.VisualStudio.Product.BuildTools', - '-requires', - 'Microsoft.VisualStudio.Component.VC.Redist.14.Latest', '-find', 'VC/Auxiliary/Build/vcvarsall.bat', '-utf8', ] + cmd += [ + '-requires', + 'Microsoft.VisualStudio.Component.VC.Redist.14.Latest', + ] + if arch.endswith('arm64'): + cmd += ['Microsoft.VisualStudio.Component.VC.Tools.ARM64'] + process = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -103,11 +112,18 @@ def get_vcvarsall(path_hint: Union[str, None] = None) -> pathlib.Path: vcvarsall = pathlib.Path(stdout.splitlines()[0]) if not vcvarsall.exists(): - raise FileNotFoundError( - 'Could not find vcvarsall.bat.' - 'Consider using --vcvarsall_path option e.g.\n' + msg = 'Could not find vcvarsall.bat.' + if arch.endswith('arm64'): + msg += ( + ' Make sure Microsoft.VisualStudio.Component.VC.Tools.ARM64 is' + ' installed.' + ) + else: + msg += ( + ' Consider using --vcvarsall_path option e.g.\n' r' --vcvarsall_path=C:\VS\VC\Auxiliary\Build\vcvarsall.bat' - ) + ) + raise FileNotFoundError(msg) return vcvarsall @@ -151,7 +167,7 @@ def get_vs_env_vars( ChildProcessError: When 'vcvarsall.bat' cannot be executed. FileNotFoundError: When 'vcvarsall.bat' cannot be found. """ - vcvarsall = get_vcvarsall(vcvarsall_path_hint) + vcvarsall = get_vcvarsall(arch, vcvarsall_path_hint) pycmd = (r'import json;' r'import os;'