diff --git a/compat/build.py b/compat/build.py index 286a64d56..549d30f5f 100755 --- a/compat/build.py +++ b/compat/build.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse +import base64 from collections import OrderedDict from dataclasses import dataclass import itertools @@ -26,9 +27,10 @@ def main(argv): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() - command = subparsers.add_parser("setup", help="setup build directories") + command = subparsers.add_parser("setup", help="setup everything needed to compile") command.add_argument("role", help="project vs subproject", choices=["project", "subproject"]) command.add_argument("builddir", help="build directory", type=Path) + command.add_argument("top_builddir", help="top build directory", type=Path) command.add_argument("host_os", help="operating system binaries are being built for") command.add_argument("host_arch", help="architecture binaries are being built for") command.add_argument("host_toolchain", help="the kind of toolchain being used", @@ -40,6 +42,7 @@ def main(argv): type=parse_array_option_value) command.set_defaults(func=lambda args: setup(args.role, args.builddir, + args.top_builddir, args.host_os, args.host_arch, args.host_toolchain, @@ -47,10 +50,10 @@ def main(argv): args.assets, args.components)) - command = subparsers.add_parser("compile", help="compile a specific build directory") - command.add_argument("builddir", help="build directory", type=Path) - command.add_argument("top_builddir", help="top build directory", type=Path) - command.set_defaults(func=lambda args: compile(args.builddir, args.top_builddir)) + command = subparsers.add_parser("compile", help="compile compatibility assets") + command.add_argument("privdir", help="directory to store intermediate files", type=Path) + command.add_argument("state", help="opaque state from the setup step") + command.set_defaults(func=lambda args: compile(args.privdir, pickle.loads(base64.b64decode(args.state)))) args = parser.parse_args() if "func" in args: @@ -82,13 +85,16 @@ def parse_array_option_value(v: str) -> set[str]: def setup(role: Role, builddir: Path, + top_builddir: Path, host_os: str, host_arch: str, host_toolchain: str, compat: set[str], assets: str, components: set[str]): - outputs: Mapping[str, Sequence[Output]] = {} + outputs: Mapping[str, Sequence[Output]] = OrderedDict() + + outputs["bundle"] = [Output("arch_support_bundle", "arch-support.bundle", Path("compat"), "")] releng_parentdir = query_releng_parentdir(role) ensure_submodules_checked_out(releng_parentdir) @@ -218,22 +224,19 @@ def setup(role: Role, target=AGENT_TARGET), ] - if not outputs: - return - - state = State(role, host_os, host_arch, host_toolchain, outputs) - privdir = compute_private_dir(builddir) - privdir.mkdir(exist_ok=True) - (privdir / STATE_FILENAME).write_bytes(pickle.dumps(state)) + state = State(role, builddir, top_builddir, host_os, host_arch, host_toolchain, outputs) + serialized_state = base64.b64encode(pickle.dumps(state)).decode('ascii') variable_names, output_names = zip(*[(output.identifier, output.name) \ for output in itertools.chain.from_iterable(outputs.values())]) - print(f"{','.join(variable_names)} {','.join(output_names)} {DEPFILE_FILENAME}") + print(f"{','.join(variable_names)} {','.join(output_names)} {DEPFILE_FILENAME} {serialized_state}") @dataclass class State: role: Role + builddir: Path + top_builddir: Path host_os: str host_arch: str host_toolchain: str @@ -248,9 +251,7 @@ class Output: target: str -def compile(builddir: Path, top_builddir: Path): - state = pickle.loads((compute_private_dir(builddir) / STATE_FILENAME).read_bytes()) - +def compile(privdir: Path, state: State): releng_parentdir = query_releng_parentdir(state.role) subprojects = detect_relevant_subprojects(releng_parentdir) if state.role == "subproject": @@ -277,12 +278,18 @@ def call_internal_meson(argv, *args, **kwargs): source_paths: set[Path] = set() options: Optional[Sequence[str]] = None build_env = scrub_environment(os.environ) - for extra_arch, outputs in state.outputs.items(): - workdir = compute_workdir_for_arch(extra_arch, builddir) + for key, outputs in state.outputs.items(): + if key == "bundle": + for o in outputs: + (state.builddir / o.name).write_bytes(b"") + continue + + extra_arch = key + workdir = (privdir / extra_arch).resolve() if not (workdir / "build.ninja").exists(): if options is None: - options = load_meson_options(top_builddir, state.role, set(subprojects.keys())) + options = load_meson_options(state.top_builddir, state.role, set(subprojects.keys())) if state.host_os == "windows": if state.host_toolchain == "microsoft": @@ -316,24 +323,16 @@ def call_internal_meson(argv, *args, **kwargs): call_meson=call_internal_meson) for o in outputs: - shutil.copy(workdir / o.file, builddir / o.name) + shutil.copy(workdir / o.file, state.builddir / o.name) for cmd in json.loads((workdir / "compile_commands.json").read_text(encoding="utf-8")): source_paths.add((workdir / Path(cmd["file"])).absolute()) - (builddir / DEPFILE_FILENAME).write_text(generate_depfile(itertools.chain.from_iterable(state.outputs.values()), - source_paths, - builddir, - top_builddir), - encoding="utf-8") - - -def compute_private_dir(builddir: Path) -> Path: - return builddir / "arch-support.p" - - -def compute_workdir_for_arch(arch: str, builddir: Path) -> Path: - return compute_private_dir(builddir) / arch + (state.builddir / DEPFILE_FILENAME).write_text(generate_depfile(itertools.chain.from_iterable(state.outputs.values()), + source_paths, + state.builddir, + state.top_builddir), + encoding="utf-8") def load_meson_options(top_builddir: Path, diff --git a/compat/meson.build b/compat/meson.build index 8feb00920..1ff7a0b7c 100644 --- a/compat/meson.build +++ b/compat/meson.build @@ -36,6 +36,7 @@ if no_overridden_compat_bits 'setup', meson.is_subproject() ? 'subproject' : 'project', meson.current_build_dir(), + meson.global_build_root(), host_os, host_abi, host_toolchain, @@ -45,64 +46,63 @@ if no_overridden_compat_bits check: true ) setup_output = setup_result.stdout().strip() - if setup_output != '' - tokens = setup_output.split(' ') - varnames = tokens[0].split(',') - outputs = tokens[1].split(',') - depfile = tokens[2] + tokens = setup_output.split(' ') + varnames = tokens[0].split(',') + outputs = tokens[1].split(',') + depfile = tokens[2] + state = tokens[3] - if host_os_family == 'darwin' - install = false - install_dir = asset_dir - else - install = true - install_dir = [] - foreach varname : varnames - if get_option('assets') == 'installed' or varname.startswith('gadget_') - install_dir += varname.endswith('_modern') ? asset_dir_modern : asset_dir_legacy - else - install_dir += false - endif - endforeach - endif + if host_os_family == 'darwin' + install = false + install_dir = asset_dir + else + install = true + install_dir = [] + foreach varname : varnames + if varname != 'arch_support_bundle' and (get_option('assets') == 'installed' or varname.startswith('gadget_')) + install_dir += varname.endswith('_modern') ? asset_dir_modern : asset_dir_legacy + else + install_dir += false + endif + endforeach + endif - arch_support = custom_target('arch-support', - output: outputs, - command: [ - build_py, - 'compile', - meson.current_build_dir(), - meson.global_build_root(), - ], - depfile: depfile, - install: install, - install_dir: install_dir, - ) + arch_support = custom_target('arch-support', + output: outputs, + command: [ + build_py, + 'compile', + '@PRIVATE_DIR@', + state, + ], + depfile: depfile, + install: install, + install_dir: install_dir, + ) - i = 0 - foreach output : arch_support.to_list() - outpath = output.full_path() - varname = varnames[i] + i = 0 + foreach output : arch_support.to_list() + outpath = output.full_path() + varname = varnames[i] - set_variable(varname, outpath) + set_variable(varname, outpath) - if varname.startswith('helper_') - helper_compat = outpath - helper_depends += arch_support - elif varname.startswith('agent_') - agent_compat = outpath - agent_depends += arch_support - elif varname.startswith('gadget_') - gadget_compat = outpath - gadget_depends += arch_support - elif varname.startswith('server_') - server_compat = outpath - server_depends += arch_support - else - assert(false, f'unexpected variable name: @varname@') - endif + if varname.startswith('helper_') + helper_compat = outpath + helper_depends += arch_support + elif varname.startswith('agent_') + agent_compat = outpath + agent_depends += arch_support + elif varname.startswith('gadget_') + gadget_compat = outpath + gadget_depends += arch_support + elif varname.startswith('server_') + server_compat = outpath + server_depends += arch_support + elif varname != 'arch_support_bundle' + assert(false, f'unexpected variable name: @varname@') + endif - i += 1 - endforeach - endif + i += 1 + endforeach endif diff --git a/src/barebone/generate-script-runtime.py b/src/barebone/generate-script-runtime.py index 588f96ba4..89ef956bd 100755 --- a/src/barebone/generate-script-runtime.py +++ b/src/barebone/generate-script-runtime.py @@ -8,22 +8,32 @@ import sys -def generate_runtime(input_dir, output_dir): - output_dir.mkdir(parents=True, exist_ok=True) +def main(argv): + input_dir, output_dir, priv_dir = [Path(d).resolve() for d in sys.argv[1:]] - shutil.copy(input_dir / "package.json", output_dir) - shutil.copy(input_dir / "package-lock.json", output_dir) + try: + generate_runtime(input_dir, output_dir, priv_dir) + except Exception as e: + print(e, file=sys.stderr) + sys.exit(1) + + +def generate_runtime(input_dir, output_dir, priv_dir): + priv_dir.mkdir(exist_ok=True) + + shutil.copy(input_dir / "package.json", priv_dir) + shutil.copy(input_dir / "package-lock.json", priv_dir) runtime_reldir = Path("script-runtime") runtime_srcdir = input_dir / runtime_reldir - runtime_intdir = output_dir / runtime_reldir + runtime_intdir = priv_dir / runtime_reldir if runtime_intdir.exists(): shutil.rmtree(runtime_intdir) shutil.copytree(runtime_srcdir, runtime_intdir) npm = os.environ.get("NPM", make_script_filename("npm")) try: - subprocess.run([npm, "install"], capture_output=True, cwd=output_dir, check=True) + subprocess.run([npm, "install"], capture_output=True, cwd=priv_dir, check=True) except Exception as e: message = "\n".join([ "", @@ -37,6 +47,8 @@ def generate_runtime(input_dir, output_dir): ]) raise EnvironmentError(message) + shutil.copy(priv_dir / "script-runtime.js", output_dir) + def make_script_filename(name): build_os = platform.system().lower() @@ -45,10 +57,4 @@ def make_script_filename(name): if __name__ == "__main__": - input_dir, output_dir = [Path(d).resolve() for d in sys.argv[1:3]] - - try: - generate_runtime(input_dir, output_dir) - except Exception as e: - print(e, file=sys.stderr) - sys.exit(1) + main(sys.argv) diff --git a/src/barebone/meson.build b/src/barebone/meson.build index 8c051f438..eecbc2171 100644 --- a/src/barebone/meson.build +++ b/src/barebone/meson.build @@ -31,6 +31,7 @@ barebone_script_runtime = custom_target('frida-barebone-script-runtime', find_program('generate-script-runtime.py'), meson.current_source_dir(), meson.current_build_dir(), + '@PRIVATE_DIR@', ], ) backend_sources += custom_target('frida-data-barebone', diff --git a/src/compiler/generate-agent.py b/src/compiler/generate-agent.py index 6d0bb56f0..8f1301005 100755 --- a/src/compiler/generate-agent.py +++ b/src/compiler/generate-agent.py @@ -20,18 +20,38 @@ ] -def generate_agent(input_dir, output_dir, host_os_family, host_arch, host_cpu_mode, v8_mksnapshot): +def main(argv): + input_dir, output_dir, priv_dir = [Path(d).resolve() for d in sys.argv[1:4]] + host_os_family, host_arch, host_cpu_mode = sys.argv[4:7] + v8_mksnapshot = sys.argv[7] + if v8_mksnapshot != "": + v8_mksnapshot = Path(v8_mksnapshot) + else: + v8_mksnapshot = None + + try: + generate_agent(input_dir, output_dir, priv_dir, host_os_family, host_arch, host_cpu_mode, v8_mksnapshot) + except subprocess.CalledProcessError as e: + print(e, file=sys.stderr) + print("Output:\n\t| " + "\n\t| ".join(e.output.strip().split("\n")), file=sys.stderr) + sys.exit(2) + except Exception as e: + print(e, file=sys.stderr) + sys.exit(1) + + +def generate_agent(input_dir, output_dir, priv_dir, host_os_family, host_arch, host_cpu_mode, v8_mksnapshot): npm = os.environ.get("NPM", make_script_filename("npm")) entrypoint = input_dir / "agent-entrypoint.js" - output_dir.mkdir(parents=True, exist_ok=True) + priv_dir.mkdir(exist_ok=True) for name in INPUTS: if name == "agent-entrypoint.js": continue shutil.copy(input_dir / name, priv_dir / name) run_kwargs = { - "cwd": output_dir, + "cwd": priv_dir, "stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "encoding": "utf-8", @@ -65,7 +85,7 @@ def generate_agent(input_dir, output_dir, host_os_family, host_arch, host_cpu_mo **run_kwargs) chunks = [] for component in components: - script = (output_dir / f"{component}.js").read_text(encoding="utf-8") + script = (priv_dir / f"{component}.js").read_text(encoding="utf-8") chunks.append(script) components_source = "\n".join(chunks) @@ -74,7 +94,7 @@ def generate_agent(input_dir, output_dir, host_os_family, host_arch, host_cpu_mo if v8_mksnapshot is not None: shutil.copy(entrypoint, agent) - (output_dir / "embed.js").write_text(components_source, encoding="utf-8") + (priv_dir / "embed.js").write_text(components_source, encoding="utf-8") subprocess.run([ v8_mksnapshot, "--turbo-instruction-scheduling", @@ -90,6 +110,7 @@ def generate_agent(input_dir, output_dir, host_os_family, host_arch, host_cpu_mo ]), encoding="utf-8") snapshot.write_bytes(b"") + def make_script_filename(name): build_os = platform.system().lower() extension = ".cmd" if build_os == "windows" else "" @@ -97,20 +118,4 @@ def make_script_filename(name): if __name__ == "__main__": - input_dir, output_dir = [Path(d).resolve() for d in sys.argv[1:3]] - host_os_family, host_arch, host_cpu_mode = sys.argv[3:6] - v8_mksnapshot = sys.argv[6] - if v8_mksnapshot != "": - v8_mksnapshot = Path(v8_mksnapshot) - else: - v8_mksnapshot = None - - try: - generate_agent(input_dir, output_dir, host_os_family, host_arch, host_cpu_mode, v8_mksnapshot) - except subprocess.CalledProcessError as e: - print(e, file=sys.stderr) - print("Output:\n\t| " + "\n\t| ".join(e.output.strip().split("\n")), file=sys.stderr) - sys.exit(2) - except Exception as e: - print(e, file=sys.stderr) - sys.exit(1) + main(sys.argv) diff --git a/src/compiler/meson.build b/src/compiler/meson.build index cf6405659..50ab68e05 100644 --- a/src/compiler/meson.build +++ b/src/compiler/meson.build @@ -30,6 +30,7 @@ if have_compiler_backend find_program('generate-agent.py'), meson.current_source_dir(), meson.current_build_dir(), + '@PRIVATE_DIR@', host_os_family, host_arch_gumjs, host_cpu_mode, diff --git a/src/embed-agent.py b/src/embed-agent.py index e9dbfb7b4..81e1989c0 100755 --- a/src/embed-agent.py +++ b/src/embed-agent.py @@ -14,6 +14,7 @@ def main(argv): resource_compiler = args.pop(0) lipo = pop_cmd_array_arg(args) output_dir = Path(args.pop(0)) + priv_dir = Path(args.pop(0)) resource_config = args.pop(0) agent_modern, agent_legacy, \ agent_emulated_modern, agent_emulated_legacy, \ @@ -24,7 +25,6 @@ def main(argv): print("At least one agent must be provided", file=sys.stderr) sys.exit(1) - priv_dir = output_dir / "frida-agent@emb" priv_dir.mkdir(exist_ok=True) embedded_assets = [] diff --git a/src/embed-helper.py b/src/embed-helper.py index 13d8731d2..1fb23b8c6 100755 --- a/src/embed-helper.py +++ b/src/embed-helper.py @@ -14,10 +14,10 @@ def main(argv): resource_compiler = args.pop(0) lipo = pop_cmd_array_arg(args) output_dir = Path(args.pop(0)) + priv_dir = Path(args.pop(0)) resource_config = args.pop(0) helper_modern, helper_legacy = [Path(p) if p else None for p in args[:2]] - priv_dir = output_dir / "frida-helper@emb" priv_dir.mkdir(exist_ok=True) embedded_assets = [] diff --git a/src/meson.build b/src/meson.build index 946925651..21ee4fe73 100644 --- a/src/meson.build +++ b/src/meson.build @@ -40,6 +40,7 @@ if have_local_backend resource_compiler, '>>>', lipo, '<<<', meson.current_build_dir(), + '@PRIVATE_DIR@', '@INPUT0@', ] @@ -449,10 +450,10 @@ if have_local_backend backend_sources += custom_target('frida-data-agent', input: 'agent.resources', output: [ + 'frida-data-agent-blob' + resource_blob_extension, 'frida-data-agent.vapi', 'frida-data-agent.h', 'frida-data-agent.c', - 'frida-data-agent-blob' + resource_blob_extension, ], command: [ python, @@ -463,6 +464,7 @@ if have_local_backend resource_compiler, '>>>', lipo, '<<<', meson.current_build_dir(), + '@PRIVATE_DIR@', '@INPUT0@', agent_modern, agent_legacy,