diff --git a/buck2/rust/.buckconfig b/buck2/rust/.buckconfig new file mode 100644 index 00000000..fc5ca33f --- /dev/null +++ b/buck2/rust/.buckconfig @@ -0,0 +1,32 @@ +[cells] +root = . +prelude = prelude +toolchains = toolchains +none = none + +[cell_aliases] +config = prelude +fbcode = none +fbsource = none +buck = none + +[external_cells] +prelude = bundled + +[parser] +target_platform_detector_spec = target:root//...->root//platforms:remote_platform + +[buck2] +digest_algorithms = SHA256 + +[buck2_re_client] +engine_address = .cluster.engflow.com +action_cache_address = .cluster.engflow.com +cas_address = .cluster.engflow.com +http_headers = + +[build] +execution_platforms = root//platforms:remote_platform + +[project] +ignore = .git diff --git a/buck2/rust/.buckroot b/buck2/rust/.buckroot new file mode 100644 index 00000000..e69de29b diff --git a/buck2/rust/.gitignore b/buck2/rust/.gitignore new file mode 100644 index 00000000..0a0ddb2e --- /dev/null +++ b/buck2/rust/.gitignore @@ -0,0 +1 @@ +/buck-out diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK new file mode 100644 index 00000000..2542c69f --- /dev/null +++ b/buck2/rust/BUCK @@ -0,0 +1,42 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rust_library( + name = "library", + srcs = glob( + ["src/**/*.rs"], + ), +) + +rust_binary( + name = "main", + srcs = glob( + ["bin/**/*.rs"], + ), + crate_root = "bin/main.rs", + deps = [":library"], +) + +rust_test( + name = "test", + srcs = glob( + ["test/**/*.rs"], + ), + deps = [":library"], + # TODO: remove these once https://github.com/facebook/buck2/pull/826 gets merged. + remote_execution_action_key_providers = select({ + "//platforms:engflow": "//platforms:remote_execution_action_keys", + "DEFAULT": None, + }), +) diff --git a/buck2/rust/README.md b/buck2/rust/README.md new file mode 100644 index 00000000..8eefd092 --- /dev/null +++ b/buck2/rust/README.md @@ -0,0 +1,60 @@ +# EngFlow RE + Buck2 Rust example + +This example demonstrates use of EngFlow RE for a simple Rust project built with [Buck2](https://github.com/facebook/buck2) using the prelude. + +It is based on two existing samples in the Buck2 upstream repo: + +* Simple rust project with prelude - https://github.com/facebook/buck2/tree/main/examples/with_prelude/rust +* Buck2 remote execution integration with EngFlow - https://github.com/facebook/buck2/tree/main/examples/remote_execution/engflow + +### Example structure + +In the `platforms` cell we specify: +In the `platforms` cell we specify: +* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`. +* The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly. +* The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `rust` preinstalled. This is because Buck2 rust rules do not include a hermetic rust toolchain. We use a bookworm image with rust pre-installed. Image details can be found in https://gallery.ecr.aws/docker/library/rust. The Dockerfile can be found in https://github.com/rust-lang/docker-rust/blob/700c4f146427808cfb1e07a646e4afabbe99da4f/stable/bullseye/Dockerfile. If a different version of `rust` or other tools is needed you should create your own image and publish it to a repo that is accessible from the cluster. + +In the `toolchains` cell we specify: + +* The remote rust toolchain `root//toolchains:remote_rust_toolchain` which is compatible with the remote execution environment. This toolchain is configured with the `rustc_target_triple` to match the remote execution environment, and the `compiler`, `clippy_driver` and `rustdoc` attrs are set to point to the location of the `rustc` binary in the image used for remote execution. +* The c++ toolchain `root//toolchains:cxx_tools_info_toolchain` that is compatible with the remote execution environment. +* The clang tools, `root//toolchains:path_clang_tools`, which is used by the c++ toolchain, and specifies the tools installed in the Docker image. +* The remote test execution toolchain, `root//toolchains:remote_test_execution_toolchain`. This toolchain defines platform options in the form of `capabilities`. Critically these include the `container-image`. This toolchain is identical to the one in the `buck2/cpp` sample in this repo. + +The `src`, `bin` and `test` cells: + +* Contain a copied version of https://github.com/facebook/buck2/tree/main/examples/with_prelude/rust that works with Buck2 and RE as configured in this sample project. + +* Key changes for remote execution in the top level `BUCK` file include setting environment variables to select the pre-installed rust toolchain in the container and set a custom value for `HOME`. This custom value is needed as `rustp` attempts to create a directory in the `HOME` location and we prefer this happen inside the `buck-out/` directory to guarantee actions do not modify any content outside of their scratch directory. + +To test the project with RE run (after setting up `.buckconfig` as indicated below): + +``` +buck2 test --remote-only //:test +``` + +You can also build the `main` for this sample by running: + +``` +buck2 build --remote-only //:main +``` + +Note the use of `--remote-only` to indicate remote execution should be used for the build / test command. + +### Relevant configs in `.buckconfig` + +The EngFlow endpoint and certificate should be configured as the +following: + +```ini +[buck2_re_client] +engine_address = .cluster.engflow.com +action_cache_address = .cluster.engflow.com +cas_address = .cluster.engflow.com +http_headers = + ``` + +To obtain the value of ``, log into https://.cluster.engflow.com/gettingstarted and obtain the value of `x-engflow-auth-token` in section `Method 2: JWT`, take note of this value. Then set `AUTH_HTTP_HEADERS` with the value `x-engflow-auth-method:jwt-v0,x-engflow-auth-token:. + +Note for CI runs, the auth method used is [Github Tokens](https://docs.engflow.com/re/config/authentication.html#github-tokens). diff --git a/buck2/rust/bin/main.rs b/buck2/rust/bin/main.rs new file mode 100644 index 00000000..54a4b8bf --- /dev/null +++ b/buck2/rust/bin/main.rs @@ -0,0 +1,3 @@ +fn main() { + library::print_hello(); +} diff --git a/buck2/rust/platforms/BUCK b/buck2/rust/platforms/BUCK new file mode 100644 index 00000000..67a4d912 --- /dev/null +++ b/buck2/rust/platforms/BUCK @@ -0,0 +1,42 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":defs.bzl", "platforms") +load(":defs.bzl", "action_keys") + +constraint_setting( + name = "re_provider" +) + +constraint_value( + name = "engflow", + constraint_setting = ":re_provider", +) + +# This platform configures details of remote execution. +platforms( + name = "remote_platform", + cpu_configuration = "config//cpu:x86_64", + os_configuration = "config//os:linux", + re_provider = ":engflow", +) + +# This action_key provides a default BuildModeInfo that is needed for RE of tests to function properly. +# The values in `cell` and `mode` can be used, in practice, to create cache silos. Any values can be given to these attributes. +action_keys( + name = "remote_execution_action_keys", + cell = "standard", + mode = "standard", + visibility = ["PUBLIC"], +) diff --git a/buck2/rust/platforms/defs.bzl b/buck2/rust/platforms/defs.bzl new file mode 100644 index 00000000..386a109e --- /dev/null +++ b/buck2/rust/platforms/defs.bzl @@ -0,0 +1,77 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This platform is essentially the same as the one provided in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/examples/remote_execution/engflow/platforms/defs.bzl +# The main difference is we enable passing CPU and OS constraints and we use the sample EngFlow RE image. +load("@prelude//:build_mode.bzl", "BuildModeInfo") + +def _platforms(ctx): + constraints = dict() + constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) + constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints) + constraints.update(ctx.attrs.re_provider[ConfigurationInfo].constraints) + configuration = ConfigurationInfo( + constraints = constraints, + values = {}, + ) + + # A bookworm image with rust pre-installed. Image details can be found in https://gallery.ecr.aws/docker/library/rust. + # Dockerfile can be found in https://github.com/rust-lang/docker-rust/blob/700c4f146427808cfb1e07a646e4afabbe99da4f/stable/bullseye/Dockerfile + image = "docker://public.ecr.aws/docker/library/rust:1.83.0-bullseye@sha256:24118f76a7da011b22a25b8e9dbdbb549ed29c1eba635d6aa4a9c9f5ed545066" + name = ctx.label.raw_target() + platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = False, + remote_enabled = True, + use_limited_hybrid = False, + remote_execution_properties = { + "container-image": image, + }, + remote_execution_use_case = "buck2-default", + # TODO: Use output_paths + remote_output_paths = "strict", + ), + ) + + return [ + DefaultInfo(), + ExecutionPlatformRegistrationInfo(platforms = [platform]), + configuration, + PlatformInfo(label = str(name), configuration = configuration), + ] + +def _action_keys(ctx): + return [ + DefaultInfo(), + BuildModeInfo(cell = ctx.attrs.cell, mode = ctx.attrs.mode), + ] + +platforms = rule( + attrs = { + "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]), + "os_configuration": attrs.dep(providers = [ConfigurationInfo]), + "re_provider": attrs.dep(providers = [ConfigurationInfo]), + }, + impl = _platforms +) + +action_keys = rule( + attrs = { + "cell": attrs.string(), + "mode": attrs.string(), + }, + impl = _action_keys +) diff --git a/buck2/rust/src/lib.rs b/buck2/rust/src/lib.rs new file mode 100644 index 00000000..84c45251 --- /dev/null +++ b/buck2/rust/src/lib.rs @@ -0,0 +1,3 @@ +pub fn print_hello() { + println!("hello world from rust toolchain"); +} diff --git a/buck2/rust/test/test.rs b/buck2/rust/test/test.rs new file mode 100644 index 00000000..c5cdb345 --- /dev/null +++ b/buck2/rust/test/test.rs @@ -0,0 +1,4 @@ +#[test] +fn test_it_doesnt_crash() { + library::print_hello(); +} diff --git a/buck2/rust/toolchains/BUCK b/buck2/rust/toolchains/BUCK new file mode 100644 index 00000000..9fc11b68 --- /dev/null +++ b/buck2/rust/toolchains/BUCK @@ -0,0 +1,72 @@ + +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("defs.bzl", "remote_rust_toolchain", "path_clang_tools") +load("@prelude//toolchains:cxx.bzl", "cxx_tools_info_toolchain") +load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain") +load("@prelude//toolchains:remote_test_execution.bzl", "remote_test_execution_toolchain") +load("@prelude//tests:test_toolchain.bzl", "noop_test_toolchain") + +remote_rust_toolchain( + name = "rust", + default_edition = "2021", + visibility = ["PUBLIC"], + rustdoc = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc", + compiler = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc", + clippy_driver = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc", + rustc_target_triple = "x86_64-unknown-linux-gnu", +) + +# Python toolchain used in scripts that bootstrap other aspects of the Buck2 prelude. +system_python_bootstrap_toolchain( + name = "python_bootstrap", + visibility = ["PUBLIC"], +) + +# Custom clang tools that use g++ for linking. +path_clang_tools( + name = "clang_tools", + visibility = ["PUBLIC"], +) + +# Custom cpp toolchain that is compatible with the remote worker environment. +cxx_tools_info_toolchain( + name = "cxx", + cxx_tools_info = ":clang_tools", + visibility = ["PUBLIC"], +) + +# Default toolchain for remote execution of tests. +# Note it defines a profile with a capability that defines the `container-image` that matches the one defined in //platforms:remote_platform. +# Capabilities are passed to the RE service to find workers that match them as Platform options. +remote_test_execution_toolchain( + name = "remote_test_execution", + visibility = ["PUBLIC"], + default_profile = "cxx_re_toolchain", + profiles = { + "cxx_re_toolchain": { + "use_case": "cxx-testing", + "capabilities": { + "container-image" : "docker://public.ecr.aws/docker/library/rust:1.83.0-bullseye@sha256:24118f76a7da011b22a25b8e9dbdbb549ed29c1eba635d6aa4a9c9f5ed545066", + }, + } + }, +) + +# In some cases the execution of test can fail looking for this `noop_test_toolchain`. +noop_test_toolchain( + name = "test", + visibility = ["PUBLIC"], +) diff --git a/buck2/rust/toolchains/defs.bzl b/buck2/rust/toolchains/defs.bzl new file mode 100644 index 00000000..e6b2a8f3 --- /dev/null +++ b/buck2/rust/toolchains/defs.bzl @@ -0,0 +1,95 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@prelude//rust:rust_toolchain.bzl", "PanicRuntime", "RustToolchainInfo") +load("@prelude//rust/tools:attrs.bzl", "internal_tool_attrs") +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerType") +load("@prelude//toolchains:cxx.bzl", "CxxToolsInfo") + +# This def is similar to the one in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/prelude/toolchains/cxx/clang/tools.bzl +# The main difference is we set linker to "g++" to match the cpp tools installed in the sample docker container. +def _path_clang_tools_impl(_ctx) -> list[Provider]: + return [ + DefaultInfo(), + CxxToolsInfo( + compiler = "clang", + compiler_type = "clang", + cxx_compiler = "clang++", + asm_compiler = "clang", + asm_compiler_type = "clang", + rc_compiler = None, + cvtres_compiler = None, + archiver = "ar", + archiver_type = "gnu", + linker = "g++", + linker_type = LinkerType("gnu"), + ), + ] + +path_clang_tools = rule( + impl = _path_clang_tools_impl, + attrs = {}, +) + +def _remote_rust_toolchain_impl(ctx): + return [ + DefaultInfo(), + RustToolchainInfo( + allow_lints = ctx.attrs.allow_lints, + clippy_driver = RunInfo(args = [ctx.attrs.clippy_driver]), + clippy_toml = ctx.attrs.clippy_toml[DefaultInfo].default_outputs[0] if ctx.attrs.clippy_toml else None, + compiler = RunInfo(args = [ctx.attrs.compiler]), + default_edition = ctx.attrs.default_edition, + panic_runtime = PanicRuntime("unwind"), + deny_lints = ctx.attrs.deny_lints, + doctests = ctx.attrs.doctests, + failure_filter_action = ctx.attrs.failure_filter_action[RunInfo], + nightly_features = ctx.attrs.nightly_features, + report_unused_deps = ctx.attrs.report_unused_deps, + rustc_action = ctx.attrs.rustc_action[RunInfo], + rustc_binary_flags = ctx.attrs.rustc_binary_flags, + rustc_flags = ctx.attrs.rustc_flags, + rustc_target_triple = ctx.attrs.rustc_target_triple, + rustc_test_flags = ctx.attrs.rustc_test_flags, + rustdoc = RunInfo(args = [ctx.attrs.rustdoc]), + rustdoc_flags = ctx.attrs.rustdoc_flags, + rustdoc_test_with_resources = ctx.attrs.rustdoc_test_with_resources[RunInfo], + rustdoc_coverage = ctx.attrs.rustdoc_coverage[RunInfo], + transitive_dependency_symlinks_tool = ctx.attrs.transitive_dependency_symlinks_tool[RunInfo], + warn_lints = ctx.attrs.warn_lints, + ), + ] + +remote_rust_toolchain = rule( + impl = _remote_rust_toolchain_impl, + attrs = internal_tool_attrs | { + "allow_lints": attrs.list(attrs.string(), default = []), + "clippy_driver": attrs.string(default = "clippy-driver"), + "clippy_toml": attrs.option(attrs.dep(providers = [DefaultInfo]), default = None), + "compiler": attrs.string(default = "rustc"), + "default_edition": attrs.option(attrs.string(), default = None), + "deny_lints": attrs.list(attrs.string(), default = []), + "doctests": attrs.bool(default = False), + "nightly_features": attrs.bool(default = False), + "report_unused_deps": attrs.bool(default = False), + "rustc_binary_flags": attrs.list(attrs.string(), default = []), + "rustc_flags": attrs.list(attrs.string(), default = []), + "rustc_target_triple": attrs.string(default = "x86_64-unknown-linux-gnu"), + "rustc_test_flags": attrs.list(attrs.string(), default = []), + "rustdoc": attrs.string(default = "rustdoc"), + "rustdoc_flags": attrs.list(attrs.string(), default = []), + "warn_lints": attrs.list(attrs.string(), default = []), + }, + is_toolchain_rule = True, +) diff --git a/infra/test-buck2.sh b/infra/test-buck2.sh index 710c9bba..6dd778bb 100755 --- a/infra/test-buck2.sh +++ b/infra/test-buck2.sh @@ -21,3 +21,9 @@ cd golang buck2 build //go:hello buck2 test //go/greeting:greeting_test cd .. + +# Run rust example +cd rust +buck2 build --remote-only //:main +buck2 test --remote-only //:test +cd ..