Refactor macro infra (#547)

Cargo.toml

@@ -1,12 +1,14 @@
 [workspace]
-members = ["crates/*", "xtask"]
+default-members = ["crates/*", "xtask"]
+members = ["crates/*", "lib/*", "xtask"]
 resolver = "2"
 
 [workspace.package]
 edition = "2021"
 
 [workspace.dependencies]
 flux-attrs = { path = "./crates/flux-attrs", version = "0.1.0" }
+flux-bin = { path = "./crates/flux-bin", version = "0.1.0" }
 flux-common = { path = "./crates/flux-common", version = "0.1.0" }
 flux-config = { path = "./crates/flux-config", version = "0.1.0" }
 flux-desugar = { path = "./crates/flux-desugar", version = "0.1.0" }
@@ -21,6 +23,7 @@ flux-rs = { path = "./crates/flux-rs", version = "0.1.0" }
 flux-syntax = { path = "./crates/flux-syntax", version = "0.1.0" }
 flux-tests = { path = "./crates/flux-tests", version = "0.1.0" }
 
+home = "0.5.5"
 itertools = "0.10.5"
 
 [workspace.lints.rust]

book/src/dev/develop.md

@@ -80,6 +80,18 @@ error[FLUX]: refinement type error
 
 You can also pass `-Ztrack-diagnostics=y` to enable it if you are not using `cargo xtask run`
 
+## Running outside the project
+
+To run Flux in a package outside the flux respo you need to install the binaries globally. You can
+do that using `cargo xtask install`. If you are continuously testing new changes it could be annoying
+to do it each time. To deal with this, you can set the `FLUX_SYSROOT` environment variable to change the
+location where `cargo-flux` and `rustc-flux` load the `flux-driver`. You can set it globally to point
+to the `target/debug` directory inside your local copy of the repo. This way you won't have to run
+`cargo xtask install` after every change and you can be sure you'll be using the latest local debug
+build. Just be aware that the `rustc-flux` and `cargo-flux` binaries are built for a specific toolchain,
+and you will get a dynamic linking error if the `flux-driver` was compiled with a different one. This
+is to say, you should at least run `cargo xtask install` every time after the toolchain is updated.
+
 ## Profiling Flux
 
 Set `FLUX_DUMP_TIMINGS=true` to have flux write timing diagnostics to `./log/timings`.

book/src/guide/install.md

@@ -29,4 +29,6 @@ Next, run the following to build and install `flux` binaries
 cargo xtask install
 ```
 
-This will install `flux-driver`, `rustc-flux` and `cargo-flux`. Note that you should not call `flux-driver` directly, but rather use `rustc-flux` and `cargo-flux`.
+This will the binaries `rustc-flux` and `cargo-flux` in your cargo home. These two binaries should be used
+respectively to run flux on either a single file or on a project using cargo. The installation process will
+also copy some files to `$HOME/.flux`.

book/src/guide/run.md

@@ -29,13 +29,6 @@ cargo-flux check
 
 in order to get `flux` to check your code.
 
-## Developing locally
-
-You can set the `FLUX_DRIVER_PATH` environment variable to `./target/debug/flux-driver` if you
-want `cargo-flux` and `rustc-flux` to use the version of `flux-driver` that is built
-when you run `cargo build`. This is useful if you want to run `cargo build` instead
-of `cargo xtask install` every time you make a change.
-
 ## A tiny example
 
 The following example declares a function `inc`

crates/flux-bin/Cargo.toml

@@ -19,7 +19,7 @@ test = false
 [dependencies]
 anyhow = "1.0.75"
 dirs = "5.0.1"
-flux-config.workspace = true
+home.workspace = true
 rust-toolchain-file = "0.1.1"
 
 [lints]

crates/flux-bin/src/bin/cargo-flux.rs

@@ -6,7 +6,7 @@ use std::{
 
 use anyhow::Result;
 use flux_bin::utils::{
-    extend_env_var_with_path, get_flux_driver_path, get_ld_library_path, get_rust_toolchain,
+    get_flux_driver_path, get_rust_toolchain, get_rustc_driver_lib_path, prepend_path_to_env_var,
     EXIT_ERR, LIB_PATH,
 };
 
@@ -24,19 +24,21 @@ fn main() {
 fn run() -> Result<i32> {
     let flux_driver_path = get_flux_driver_path()?;
     let rust_toolchain = get_rust_toolchain()?;
-    let ld_library_path = get_ld_library_path(&rust_toolchain)?;
-    let extended_lib_path = extend_env_var_with_path(LIB_PATH, ld_library_path)?;
+    let ld_library_path = get_rustc_driver_lib_path(&rust_toolchain)?;
+    let extended_lib_path = prepend_path_to_env_var(LIB_PATH, ld_library_path)?;
 
     let cargo_target = env::var("CARGO_TARGET_DIR").unwrap_or_else(|_| "target".to_string());
     let cargo_target = PathBuf::from_iter([cargo_target, "flux".to_string()]);
 
-    let features = ["--features", "flux-rs/enabled"].iter();
-
     let exit_code = Command::new("cargo")
         // Skip the invocation of cargo-flux itself
         .args(env::args().skip(1))
-        .args(features)
         .env(LIB_PATH, extended_lib_path)
+        // CODESYNC(build-sysroot, 5) When running flux within cargo we want to compile flux libraries
+        // as the precompiled versions.
+        .env("FLUX_BUILD_SYSROOT", "1")
+        // CODESYNC(flux-cargo) Tell the flux-driver it's being called from cargo.
+        .env("FLUX_CARGO", "1")
         .env("RUST_TOOLCHAIN", rust_toolchain.clone())
         .env("RUSTUP_TOOLCHAIN", rust_toolchain)
         .env("RUSTC", flux_driver_path)

crates/flux-bin/src/bin/rustc-flux.rs

@@ -5,8 +5,8 @@ use std::{
 
 use anyhow::Result;
 use flux_bin::utils::{
-    extend_env_var_with_path, get_flux_driver_path, get_ld_library_path, get_rust_toolchain,
-    EXIT_ERR, LIB_PATH,
+    get_flux_driver_path, get_rust_toolchain, get_rustc_driver_lib_path, prepend_path_to_env_var,
+    sysroot_dir, EXIT_ERR, LIB_PATH,
 };
 
 fn main() {
@@ -23,12 +23,18 @@ fn main() {
 fn run() -> Result<i32> {
     let flux_driver_path = get_flux_driver_path()?;
     let rust_toolchain = get_rust_toolchain()?;
-    let ld_library_path = get_ld_library_path(&rust_toolchain)?;
-    let extended_lib_path = extend_env_var_with_path(LIB_PATH, ld_library_path)?;
+    let ld_library_path = get_rustc_driver_lib_path(&rust_toolchain)?;
+    let extended_lib_path = prepend_path_to_env_var(LIB_PATH, ld_library_path)?;
 
     let exit_code = Command::new(flux_driver_path)
         // Skip the invocation of rustc-flux itself
         .args(env::args().skip(1))
+        .arg("-Zcrate-attr=feature(register_tool,custom_inner_attributes)")
+        .arg("-Zcrate-attr=register_tool(flux)")
+        .arg("-L")
+        .arg(sysroot_dir())
+        .arg("--extern")
+        .arg("flux_rs")
         .env(LIB_PATH, extended_lib_path)
         .status()?
         .code();

crates/flux-bin/src/utils.rs

@@ -1,7 +1,6 @@
 use std::{env, ffi::OsString, fs, path::PathBuf};
 
 use anyhow::{anyhow, Result};
-use flux_config as config;
 
 #[cfg(target_os = "windows")]
 pub const LIB_PATH: &str = "PATH";
@@ -12,26 +11,29 @@ pub const LIB_PATH: &str = "DYLD_FALLBACK_LIBRARY_PATH";
 
 pub const EXIT_ERR: i32 = -1;
 
-pub fn get_default_flux_driver_path() -> Result<PathBuf> {
-    let mut default_flux_path =
-        env::current_exe().map(|path| path.with_file_name("flux-driver"))?;
-    if cfg!(target_os = "windows") {
-        default_flux_path.set_extension("exe");
-    }
-    Ok(default_flux_path)
+pub const FLUX_SYSROOT: &str = "FLUX_SYSROOT";
+
+/// The path of the flux sysroot lib containing precompiled libraries and the flux driver.
+pub fn sysroot_dir() -> PathBuf {
+    env::var(FLUX_SYSROOT).map_or_else(|_| default_sysroot_dir(), PathBuf::from)
+}
+
+/// Return the default sysroot
+pub fn default_sysroot_dir() -> PathBuf {
+    home::home_dir()
+        .expect("Couldn't find home directory")
+        .join(".flux")
 }
 
 pub fn get_flux_driver_path() -> Result<PathBuf> {
-    let flux_driver_path = config::driver_path()
-        .cloned()
-        .map_or_else(get_default_flux_driver_path, Ok)?;
-    if !flux_driver_path.is_file() {
-        return Err(anyhow!(
-            "flux executable {:?} does not exist or is not a file",
-            flux_driver_path
-        ));
+    let mut path = sysroot_dir().join("flux-driver");
+    if cfg!(target_os = "windows") {
+        path.set_extension("exe");
     }
-    Ok(flux_driver_path)
+    if !path.is_file() {
+        return Err(anyhow!("flux executable {:?} does not exist or is not a file", path));
+    }
+    Ok(path)
 }
 
 pub fn get_rust_toolchain() -> Result<String> {
@@ -46,9 +48,9 @@ pub fn get_rust_toolchain() -> Result<String> {
         .map(|channel| channel.name().to_string())
 }
 
-pub fn get_ld_library_path(rust_toolchain: &str) -> Result<PathBuf> {
-    let rustup_home_path = get_rustup_home()?;
-    let toolchains_path = rustup_home_path.join("toolchains");
+/// Path from where to load the rustc-driver library from
+pub fn get_rustc_driver_lib_path(rust_toolchain: &str) -> Result<PathBuf> {
+    let toolchains_path = home::rustup_home()?.join("toolchains");
     if toolchains_path.is_dir() {
         let entries = fs::read_dir(toolchains_path)?;
         for entry in entries {
@@ -68,30 +70,13 @@ pub fn get_ld_library_path(rust_toolchain: &str) -> Result<PathBuf> {
     Err(anyhow!("Could not read Rustup toolchains folder"))
 }
 
-pub fn get_rustup_home() -> Result<PathBuf> {
-    env::var("RUSTUP_HOME").map(PathBuf::from).or_else(|e| {
-        match e {
-            env::VarError::NotPresent => {
-                dirs::home_dir()
-                    .ok_or_else(|| anyhow!("Could not get OS's home dir"))
-                    .map(|home_dir| home_dir.join(".rustup"))
-            }
-            _ => Err(anyhow::Error::from(e)),
-        }
-    })
-}
-
 /// Prepends the path so that it's the first checked.
-pub fn extend_env_var_with_path(var_name: &str, new_path: PathBuf) -> Result<OsString> {
-    let mut paths = env::var(var_name)
-        .map(|paths_str| env::split_paths(&paths_str).collect())
-        .or_else(|err| {
-            match err {
-                env::VarError::NotPresent => Ok(Vec::new()),
-                e => Err(e),
-            }
-        })?;
-    // clone the path so we can report it in the error message.
+pub fn prepend_path_to_env_var(var_name: &str, new_path: PathBuf) -> Result<OsString> {
+    let mut paths = match env::var(var_name) {
+        Ok(paths) => env::split_paths(&paths).collect(),
+        Err(env::VarError::NotPresent) => vec![],
+        Err(e) => Err(e)?,
+    };
     paths.insert(0, new_path);
     env::join_paths(paths).map_err(anyhow::Error::from)
 }

crates/flux-config/src/lib.rs

@@ -53,10 +53,6 @@ pub fn cache_path() -> PathBuf {
     log_dir().join(&CONFIG.cache_file)
 }
 
-pub fn driver_path() -> Option<&'static PathBuf> {
-    CONFIG.driver_path.as_ref()
-}
-
 pub fn check_overflow() -> bool {
     CONFIG.check_overflow
 }
@@ -73,7 +69,6 @@ pub struct CrateConfig {
 
 #[derive(Deserialize)]
 struct Config {
-    driver_path: Option<PathBuf>,
     log_dir: PathBuf,
     dump_constraint: bool,
     dump_checker_trace: bool,

crates/flux-driver/src/bin/main.rs

@@ -12,8 +12,8 @@ mod logger;
 fn main() -> io::Result<()> {
     let resolve_logs = logger::install()?;
 
-    // Check if we are being called from cargo.
-    let in_cargo = env::var("CARGO").is_ok();
+    // CODESYNC(flux-cargo) Check if we are being called from cargo.
+    let in_cargo = env::var("FLUX_CARGO").is_ok();
 
     // HACK(nilehmann)
     // Disable incremental compilation because that makes the borrow checker to not run

crates/flux-macros/src/diagnostics/diagnostic.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::pedantic)]
 #![deny(unused_must_use)]
 
 use std::cell::RefCell;

crates/flux-macros/src/diagnostics/diagnostic_builder.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::pedantic)]
 #![deny(unused_must_use)]
 
 use proc_macro2::{Ident, Span, TokenStream};

crates/flux-macros/src/diagnostics/fluent.rs

@@ -1,4 +1,4 @@
-#![allow(clippy::all)]
+#![allow(clippy::all, clippy::pedantic)]
 use std::{
     collections::{HashMap, HashSet},
     fs::read_to_string,

crates/flux-macros/src/diagnostics/subdiagnostic.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::pedantic)]
 #![deny(unused_must_use)]
 
 use proc_macro2::TokenStream;