parse project wide metadata

.gitignore

@@ -1,2 +1,2 @@
-/Cargo.lock
-/target
+Cargo.lock
+target

Cargo.toml

@@ -1,5 +1,8 @@
-[package]
-name = "system-deps"
+[workspace]
+members = [ "meta" ]
+exclude = [ "target" ]
+
+[workspace.package]
 version = "7.0.3"
 authors = [
     "Guillaume Desmottes <[email protected]>",
@@ -17,6 +20,19 @@ keywords = [
 ]
 edition = "2018"
 documentation = "https://docs.rs/system-deps/"
+
+[workspace.dependencies]
+system-deps-meta = { path = "./meta" }
+
+[package]
+name = "system-deps"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+description.workspace = true
+keywords.workspace = true
+edition.workspace = true
+documentation.workspace = true
 readme = "README.md"
 
 [dependencies]

meta/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "system-deps-meta"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+description.workspace = true
+keywords.workspace = true
+edition.workspace = true
+documentation.workspace = true
+
+[dependencies]
+cargo_metadata = "0.19"
+serde = "1.0"
+toml = "0.8"
+cfg-expr = { version = "0.17", features = ["targets"] }
+sha256 = { version = "1.5", optional = true }
+attohttpc = { version = "0.28", optional = true }
+flate2 = { version = "1.0", optional = true }
+xz = { version = "0.1", optional = true }
+tar = { version = "0.4", optional = true }
+zip = { version = "2.2", optional = true }
+
+[features]
+binary = [ "dep:sha256", "dep:attohttpc" ]
+gz = [ "dep:flate2", "dep:tar" ]
+xz = [ "dep:xz", "dep:tar" ]
+zip = [ "dep:zip" ]
+test = [ ]

meta/build.rs

@@ -0,0 +1,80 @@
+use std::{
+    env,
+    path::{Path, PathBuf},
+};
+
+/// Environment variable to override the top level `Cargo.toml`.
+const MANIFEST_VAR: &str = "SYSTEM_DEPS_BUILD_MANIFEST";
+
+/// Environment variable to override the directory where `system-deps`
+/// will store build products such as binary outputs.
+const TARGET_VAR: &str = "SYSTEM_DEPS_TARGET_DIR";
+
+/// Try to find the project root using locate-project
+fn find_with_cargo(dir: &Path) -> Option<PathBuf> {
+    let out = std::process::Command::new(env!("CARGO"))
+        .current_dir(dir)
+        .arg("locate-project")
+        .arg("--workspace")
+        .arg("--message-format=plain")
+        .output()
+        .ok()?
+        .stdout;
+    if out.is_empty() {
+        return None;
+    }
+    Some(PathBuf::from(std::str::from_utf8(&out).ok()?.trim()))
+}
+
+/// Get the manifest from the project directory. This is **not** the directory
+/// where `system-deps` is cloned, it should point to the top level `Cargo.toml`
+/// file. This is needed to obtain metadata from all of dependencies, including
+/// those downstream of the package being compiled.
+///
+/// If the target directory is not a subfolder of the project it will not be
+/// possible to detect it automatically. In this case, the user will be asked
+/// to specify the `SYSTEM_DEPS_MANIFEST` variable to point to it.
+///
+/// See https://github.com/rust-lang/cargo/issues/3946 for updates on first
+/// class support for finding the workspace root.
+fn manifest() -> PathBuf {
+    println!("cargo:rerun-if-env-changed={}", MANIFEST_VAR);
+    if let Ok(root) = env::var(MANIFEST_VAR) {
+        return PathBuf::from(&root);
+    }
+
+    // When build scripts are invoked, they have one argument pointing to the
+    // build path of the crate in the target directory. This is different than
+    // the `OUT_DIR` environment variable, that can point to a target directory
+    // where the checkout of the dependency is.
+    let mut dir = PathBuf::from(
+        std::env::args()
+            .next()
+            .expect("There should be cargo arguments for determining the root"),
+    );
+    dir.pop();
+
+    // Try to find the project with cargo
+    find_with_cargo(&dir).expect(
+        "Error determining the cargo root manifest.\n\
+         Please set `SYSTEM_DEPS_MANIFEST` to the path of your project's Cargo.toml",
+    )
+}
+
+/// Set compile time values for the manifest and target paths, and the compile target.
+/// Calculating this in a build script is necessary so that they are only calculated
+/// once and every invocation of `system-deps` references the same metadata.
+pub fn main() {
+    let manifest = manifest();
+    println!("cargo:rerun-if-changed={}", manifest.display());
+    println!("cargo:rustc-env=BUILD_MANIFEST={}", manifest.display());
+
+    let target_dir = env::var(TARGET_VAR).or(env::var("OUT_DIR")).unwrap();
+    println!("cargo:rerun-if-env-changed={}", TARGET_VAR);
+    println!("cargo:rustc-env=BUILD_TARGET_DIR={}", target_dir);
+
+    println!(
+        "cargo:rustc-env=TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}

meta/src/error.rs

@@ -0,0 +1,50 @@
+use std::fmt;
+
+/// Metadata parsing errors.
+#[derive(Debug)]
+pub enum Error {
+    /// The toml object guarded by the cfg() expression is too shallow.
+    CfgNotObject(String),
+    /// Error while deserializing metadata.
+    DeserializeError(toml::de::Error),
+    /// Merging two incompatible branches.
+    IncompatibleMerge,
+    /// Error while parsing the cfg() expression.
+    InvalidCfg(cfg_expr::ParseError),
+    /// Tried to find the package but it is not in the metadata tree.
+    PackageNotFound(String),
+    /// Error while deserializing metadata.
+    SerializeError(toml::ser::Error),
+    /// The cfg() expression is valid, but not currently supported.
+    UnsupportedCfg(String),
+}
+
+impl From<toml::de::Error> for Error {
+    fn from(e: toml::de::Error) -> Self {
+        Self::DeserializeError(e)
+    }
+}
+
+impl From<toml::ser::Error> for Error {
+    fn from(e: toml::ser::Error) -> Self {
+        Self::SerializeError(e)
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::CfgNotObject(s) => {
+                write!(f, "The expression '{}' is not guarding a package", s)
+            }
+            Self::DeserializeError(e) => write!(f, "Error while parsing: {}", e),
+            Self::IncompatibleMerge => write!(f, "Can't merge metadata"),
+            Self::PackageNotFound(s) => write!(f, "Package not found: {}", s),
+            Self::SerializeError(e) => write!(f, "Error while parsing: {}", e),
+            Self::UnsupportedCfg(s) => {
+                write!(f, "Unsupported cfg() expression: {}", s)
+            }
+            e => e.fmt(f),
+        }
+    }
+}

meta/src/lib.rs

@@ -0,0 +1,14 @@
+//#![warn(missing_docs)]
+
+pub mod error;
+pub mod parse;
+pub mod utils;
+
+#[cfg(any(test, feature = "test"))]
+pub mod test;
+
+/// Path to the top level Cargo.toml.
+pub const BUILD_MANIFEST: &str = env!("BUILD_MANIFEST");
+
+/// Directory where `system-deps` related build products will be stored.
+pub const BUILD_TARGET_DIR: &str = env!("BUILD_TARGET_DIR");

meta/src/parse.rs

@@ -0,0 +1,169 @@
+use std::{
+    collections::{BTreeSet, HashMap, HashSet, VecDeque},
+    iter,
+    path::Path,
+};
+
+use cargo_metadata::{DependencyKind, MetadataCommand};
+use serde::Serialize;
+use toml::Table;
+
+use crate::{error::Error, utils::reduce};
+
+/// Stores a section of metadata found in one package.
+#[derive(Clone, Debug, Default, Serialize)]
+pub struct MetadataNode {
+    /// Deserialized metadata.
+    table: Table,
+    /// The parents of this package.
+    parents: BTreeSet<String>,
+    /// The number of children.
+    children: usize,
+}
+
+/// Recursively read dependency manifests to find metadata matching a key using cargo_metadata.
+///
+/// ```toml
+/// [package.metadata.section]
+/// some_value = ...
+/// other_value = ...
+/// ```
+pub fn read_metadata(
+    manifest: impl AsRef<Path>,
+    section: &str,
+    merge: impl Fn(&mut Table, Table, bool) -> Result<(), Error>,
+) -> Result<Table, Error> {
+    let data = MetadataCommand::new()
+        .manifest_path(manifest.as_ref())
+        .exec()
+        .unwrap();
+
+    // Create the root node from the workspace metadata
+    let value = data.workspace_metadata.get(section).cloned();
+    let root_node = MetadataNode {
+        table: reduce(
+            value
+                .and_then(|v| Table::try_from(v).ok())
+                .unwrap_or_default(),
+        )?,
+        ..Default::default()
+    };
+
+    // Use the root package or all the workspace packages as a starting point
+    let mut packages: VecDeque<_> = if let Some(root) = data.root_package() {
+        [(root, "")].into()
+    } else {
+        data.workspace_packages()
+            .into_iter()
+            .zip(iter::repeat(""))
+            .collect()
+    };
+
+    let mut nodes = HashMap::from([("", root_node)]);
+
+    // Iterate through the dependency tree to visit all packages
+    let mut visited = HashSet::new();
+    while let Some((pkg, parent)) = packages.pop_front() {
+        let name = pkg.name.as_str();
+
+        // If we already handled this node, update parents and keep going
+        if !visited.insert(name) {
+            if let Some(node) = nodes.get_mut(name) {
+                if node.parents.insert(parent.into()) {
+                    if let Some(p) = nodes.get_mut(parent) {
+                        p.children += 1
+                    }
+                }
+            }
+            continue;
+        }
+
+        // Keep track of the local manifests to see if they change
+        if pkg
+            .manifest_path
+            .starts_with(manifest.as_ref().parent().unwrap())
+        {
+            println!("cargo:rerun-if-changed={}", pkg.manifest_path);
+        };
+
+        // Get `package.metadata.section` and add it to the metadata graph
+        let node = match (nodes.get_mut(name), pkg.metadata.get(section).cloned()) {
+            (None, Some(s)) => {
+                let node = MetadataNode {
+                    table: reduce(Table::try_from(s)?)?,
+                    ..Default::default()
+                };
+                nodes.insert(name, node);
+                nodes.get_mut(name)
+            }
+            (n, _) => n,
+        };
+
+        // Update parents
+        let next_parent = if let Some(node) = node {
+            if node.parents.insert(parent.into()) {
+                if let Some(p) = nodes.get_mut(parent) {
+                    p.children += 1
+                }
+            }
+            name
+        } else {
+            parent
+        };
+
+        // Add dependencies to the queue
+        for dep in &pkg.dependencies {
+            if !matches!(dep.kind, DependencyKind::Normal) {
+                continue;
+            }
+            if let Some(dep_pkg) = data.packages.iter().find(|p| p.name == dep.name) {
+                packages.push_back((dep_pkg, next_parent));
+            };
+        }
+    }
+
+    // Now that the tree is built, apply the reducing rules
+    let mut res = Table::new();
+    let mut curr = Table::new();
+
+    // Initialize the queue from the leaves
+    // NOTE: Use `extract_if` when it is available https://github.com/rust-lang/rust/issues/43244
+    let mut queue = VecDeque::new();
+    let mut nodes: HashMap<&str, MetadataNode> = nodes
+        .into_iter()
+        .filter_map(|(k, v)| {
+            if v.children == 0 {
+                queue.push_back(v);
+                None
+            } else {
+                Some((k, v))
+            }
+        })
+        .collect();
+
+    while let Some(node) = queue.pop_front() {
+        // Push the parents to the queue, avoid unnecessary clones
+        for p in node.parents.iter().rev() {
+            let Some(parent) = nodes.get_mut(p.as_str()) else {
+                return Err(Error::PackageNotFound(p.into()));
+            };
+            let next = if parent.children.checked_sub(1).is_some() {
+                println!("cargo:warning=clone");
+                parent.clone()
+            } else {
+                nodes.remove(p.as_str()).expect("Already checked")
+            };
+            queue.push_front(next);
+        }
+
+        let reduced = reduce(node.table)?;
+        merge(&mut curr, reduced, true)?;
+
+        if node.parents.is_empty() {
+            merge(&mut res, curr, false)?;
+            curr = Table::new();
+        }
+    }
+
+    Ok(res)
+}