From f94331974128845feebe5b7fc486f7eeb0ec53e9 Mon Sep 17 00:00:00 2001 From: eri Date: Fri, 13 Dec 2024 17:54:25 +0100 Subject: [PATCH] support prebuilt library binaries --- Cargo.toml | 15 +- build.rs | 32 ++ meta/src/binary.rs | 383 +++++++++++++ meta/src/error.rs | 83 +++ meta/src/lib.rs | 3 + src/lib.rs | 246 ++++++-- src/metadata.rs | 5 + src/test.rs | 44 +- src/test_binary.rs | 589 ++++++++++++++++++++ src/tests/test.tar.gz | Bin 0 -> 344 bytes src/tests/test.tar.xz | Bin 0 -> 388 bytes src/tests/test.zip | Bin 0 -> 784 bytes src/tests/uninstalled/info.toml | 1 + src/tests/uninstalled/lib/libtest.a | 0 src/tests/uninstalled/lib/pkgconfig/test.pc | 9 + 15 files changed, 1352 insertions(+), 58 deletions(-) create mode 100644 build.rs create mode 100644 meta/src/binary.rs create mode 100644 src/test_binary.rs create mode 100644 src/tests/test.tar.gz create mode 100644 src/tests/test.tar.xz create mode 100644 src/tests/test.zip create mode 100644 src/tests/uninstalled/info.toml create mode 100644 src/tests/uninstalled/lib/libtest.a create mode 100644 src/tests/uninstalled/lib/pkgconfig/test.pc diff --git a/Cargo.toml b/Cargo.toml index 656c213..a8447bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,13 +36,26 @@ documentation.workspace = true readme = "README.md" [dependencies] +system-deps-meta = { workspace = true, optional = true } pkg-config = "0.3.25" toml = { version = "0.8", default-features = false, features = ["parse"] } version-compare = "0.2" heck = "0.5" cfg-expr = { version = "0.17", features = ["targets"] } +[build-dependencies] +system-deps-meta = { workspace = true, optional = true } + [dev-dependencies] -lazy_static = "1" +system-deps-meta = { workspace = true, features = ["test"] } itertools = "0.13" assert_matches = "1.5" +tiny_http = "0.12" + +[features] +default = [ ] +# How to do this using resolver v2? Since features are separated +binary = [ "system-deps-meta/binary" ] +gz = [ "system-deps-meta/gz" ] +xz = [ "system-deps-meta/xz" ] +zip = [ "system-deps-meta/zip" ] diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..d7c1679 --- /dev/null +++ b/build.rs @@ -0,0 +1,32 @@ +pub fn main() { + #[cfg(feature = "binary")] + binary::build().unwrap_or_else(|e| panic!("{}", e)); +} + +#[cfg(feature = "binary")] +mod binary { + use std::{fs, path::Path}; + + use system_deps_meta::{ + binary::{merge_binary, Paths}, + error::{BinaryError, Error}, + parse::read_metadata, + BUILD_MANIFEST, BUILD_TARGET_DIR, + }; + + // Add pkg-config paths to the overrides + pub fn build() -> Result<(), Error> { + // Read metadata from the crate graph + let metadata = read_metadata(BUILD_MANIFEST, "system-deps", merge_binary)?; + + // Download the binaries and get their pkg_config paths + let paths: Paths = metadata.into_iter().collect(); + + // Write the binary paths to a file for later use + let dest = Path::new(BUILD_TARGET_DIR).join("paths.toml"); + fs::write(&dest, paths.to_string()?).map_err(BinaryError::InvalidDirectory)?; + println!("cargo:rustc-env=BINARY_PATHS={}", dest.display()); + + Ok(()) + } +} diff --git a/meta/src/binary.rs b/meta/src/binary.rs new file mode 100644 index 0000000..4fd3681 --- /dev/null +++ b/meta/src/binary.rs @@ -0,0 +1,383 @@ +use std::{ + collections::HashMap, + convert::{TryFrom, TryInto}, + fs, + iter::FromIterator, + path::{Path, PathBuf}, + sync::{Mutex, OnceLock}, + thread, +}; + +use serde::{Deserialize, Serialize}; +use toml::{Table, Value}; + +use crate::{ + error::{BinaryError, Error}, + utils::merge_default, +}; + +/// The extension of the binary archive. +/// Support for different extensions is enabled using features. +#[derive(Debug)] +pub enum Extension { + /// A `.tar.gz` archive. + #[cfg(feature = "gz")] + TarGz, + /// A `.tar.xz` archive. + #[cfg(feature = "xz")] + TarXz, + /// A `.zip` archive. + #[cfg(feature = "zip")] + Zip, + Folder, +} + +impl TryFrom<&str> for Extension { + type Error = BinaryError; + fn try_from(value: &str) -> Result { + Path::new(value).try_into() + } +} + +impl TryFrom<&Path> for Extension { + type Error = BinaryError; + fn try_from(path: &Path) -> Result { + if path.is_dir() { + return Ok(Self::Folder); + }; + let Some(ext) = path.extension() else { + return Err(BinaryError::UnsupportedExtension("".into())); + }; + match ext { + #[cfg(feature = "gz")] + e if e == "gz" || e == "tgz" => Ok(Extension::TarGz), + #[cfg(feature = "xz")] + e if e == "xz" => Ok(Extension::TarXz), + #[cfg(feature = "zip")] + e if e == "zip" => Ok(Extension::Zip), + e => Err(BinaryError::UnsupportedExtension( + e.to_str().unwrap().into(), + )), + } + } +} + +/// Binary locations can be specified either by describing its metadata or by refering to another +/// package. This helper enum allows deserializing both as valid versions. +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum Binary { + Follow(FollowBinary), + Url(UrlBinary), +} + +impl TryFrom for Binary { + type Error = Error; + fn try_from(value: Value) -> Result { + Ok(value.try_into()?) + } +} + +/// A package that doesn't point to a binary itself but instead uses the metadata from another one. +/// While the `follows` field can be specified, it is usually more convenient to use `provides` in +/// the package that defines the binary url. Both of these are equivalent: +/// +/// ```toml +/// [package.metadata.system-deps.a] +/// url = "..." +/// provides = [ "b" ] +/// +/// [package.metadata.system-deps.b] +/// follows = "a" +/// ``` +/// +/// Specifying both an url and a followed package is incompatible and it will cause an error. +#[derive(Debug, Deserialize)] +pub struct FollowBinary { + /// The package name to get the metadata from. + follows: String, +} + +/// Represents one location from where to download prebuilt binaries. +#[derive(Debug, Deserialize)] +pub struct UrlBinary { + /// The url from which to download the archived binaries. It suppports: + /// + /// - Web urls, in the form `http[s]://website/archive.ext`. + /// This must directly download an archive with a known `Extension`. + /// - Local files, in the form `file:///path/to/archive.ext`. + /// Note that this is made of the url descriptor `file://`, and then an absolute path, that + /// starts with `/`, so three total slashes are needed. + /// The path can point at an archive with a known `Extension`, or to a folder containing the + /// uncompressed binaries. + url: String, + /// Optionally, a checksum of the downloaded archive. When set, it is used to correctly cache + /// the result. If this is not specified, it will still be cached by cargo, but redownloads + /// might happen more often. It has no effect if `url` is a local folder. + checksum: Option, + /// A list of relative paths inside the binary archive that point to a folder containing + /// package config files. These directories will be prepended to the `PKG_CONFIG_PATH` when + /// compiling the affected libraries. + paths: Option>, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct Paths { + paths: HashMap>, + follows: HashMap, + wildcards: HashMap, +} + +impl FromIterator<(String, T)> for Paths +where + Binary: TryFrom, +{ + /// Uses the metadata from the cargo manifests and the environment to build a list of urls + /// from where to download binaries for dependencies and adds them to their `PKG_CONFIG_PATH`. + /// + /// This function may panic, but only on unrecoverable results such as downloading or + /// decompressing errors. While it would possible to pass these values to the caller, in this + /// particular instance it would be hard to use this trait and it complicates error management. + fn from_iter>(binaries: I) -> Self { + let mut res = Self::default(); + + let (url_binaries, follow_binaries): (Vec<_>, Vec<_>) = binaries + .into_iter() + .filter_map(|(k, v)| Some((k, v.try_into().ok()?))) + .partition(|(_, bin)| matches!(bin, Binary::Url(_))); + + // Binaries with its own url + thread::scope(|s| { + for (name, bin) in url_binaries { + let Binary::Url(bin) = bin else { + unreachable!(); + }; + + let dst = Path::new(&crate::BUILD_TARGET_DIR).join(&name); + res.paths.insert( + name, + bin.paths.iter().flatten().map(|p| dst.join(p)).collect(), + ); + + // Only refresh the binaries if there isn't already a valid copy + let valid = check_valid_dir(&dst, bin.checksum.as_deref()) + .unwrap_or_else(|e| panic!("{}", e)); + + // Allow multiple downloads at the same time + if !valid { + s.spawn(move || make_available(bin, &dst).map_err(|e| panic!("{}", e))); + } + } + }); + + // Check if the package provided extra configuration + for (name, list) in res.paths.iter_mut() { + let dst = Path::new(&crate::BUILD_TARGET_DIR).join(name); + let Ok(info) = fs::read_to_string(dst.join("info.toml")) else { + continue; + }; + let Ok(table) = toml::from_str::(&info) else { + continue; + }; + if let Some(Value::Array(paths)) = table.get("paths") { + for p in paths.iter().filter_map(|p| p.as_str()) { + let p = dst.join(p); + if !list.contains(&p) { + list.push(p); + } + } + } + } + + // Binaries that follow others + for (name, bin) in follow_binaries { + let Binary::Follow(bin) = bin else { + unreachable!(); + }; + if !res.paths.contains_key(&bin.follows) { + panic!("{}", BinaryError::InvalidFollows(name, bin.follows)); + }; + match name.strip_suffix("*") { + Some(wildcard) => res.wildcards.insert(wildcard.into(), bin.follows), + None => res.follows.insert(name, bin.follows), + }; + } + + res + } +} + +impl Paths { + /// Returns the list of paths for a certain package. Matches wildcards but they never have + /// priority over explicit urls or follows, even if they are defined higher in the hierarchy. + pub fn get(&self, key: &str) -> Option<&Vec> { + if let Some(paths) = self.paths.get(key) { + return Some(paths); + }; + + if let Some(follows) = self.follows.get(key) { + return self.paths.get(follows); + }; + + self.wildcards.iter().find_map(|(k, v)| { + key.starts_with(k) + .then_some(v) + .and_then(|v| self.paths.get(v)) + }) + } + + /// Serializes the path list. + pub fn to_string(&self) -> Result { + Ok(toml::to_string(self)?) + } +} + +/// Checks if the target directory is valid and if binaries need to be redownloaded. +/// On an `Ok` result, if the value is true it means that the directory is correct. +fn check_valid_dir(dst: &Path, checksum: Option<&str>) -> Result { + // If it doesn't exist yet the download will need to happen + if !dst.try_exists().map_err(BinaryError::InvalidDirectory)? { + return Ok(false); + } + + // Raise an error if it is a file + if dst.is_file() { + return Err(BinaryError::DirectoryIsFile(dst.display().to_string())); + } + + // Check if the checksum is valid + // If a checksum is not specified, assume the directory is invalid + if let Some(ch) = checksum { + let file = dst.join("checksum"); + Ok(file.is_file() + && ch == fs::read_to_string(file).map_err(BinaryError::InvalidDirectory)?) + } else { + Ok(false) + } +} + +/// Retrieve a binary archive from the specified `url` and decompress it in the target directory. +/// "Download" is used as an umbrella term, since this can also be a local file. +fn make_available(bin: UrlBinary, dst: &Path) -> Result<(), BinaryError> { + // TODO: Find a way of printing download/decompress progress + static LOCK: OnceLock> = OnceLock::new(); + + // Check whether the file is local or not + let (url, local) = match bin.url.strip_prefix("file://") { + Some(file) => (file, true), + None => (bin.url.as_str(), false), + }; + + let ext = url.try_into()?; + + // Check if it is a folder and it can be symlinked + if matches!(ext, Extension::Folder) { + if !local { + return Err(BinaryError::UnsupportedExtension("".into())); + } + let _l = LOCK.get_or_init(|| Mutex::new(())).lock(); + if !dst.read_link().is_ok_and(|l| l == Path::new(url)) { + if dst.is_symlink() { + std::fs::remove_file(dst).map_err(BinaryError::SymlinkError)?; + } + #[cfg(unix)] + std::os::unix::fs::symlink(url, dst).map_err(BinaryError::SymlinkError)?; + #[cfg(windows)] + std::os::windows::fs::symlink_dir(url, dst).map_err(BinaryError::SymlinkError)?; + } + return Ok(()); + } + + // Otherwise, use a local file or download from the web + let file = if local { + fs::read(url).map_err(BinaryError::LocalFileError)? + } else { + let res = attohttpc::get(url).send()?; + res.error_for_status()?.bytes()? + }; + + // Verify the checksum + let calculated = sha256::digest(&*file); + let checksum = match bin.checksum { + Some(ch) if *ch == calculated => Ok(ch), + _ => Err(BinaryError::InvalidChecksum( + url.into(), + bin.checksum.unwrap_or("".into()), + calculated, + )), + }?; + fs::create_dir_all(dst).map_err(BinaryError::DecompressError)?; + fs::write(dst.join("checksum"), checksum).map_err(BinaryError::DecompressError)?; + + // Decompress the binary archive + decompress(&file, dst, ext)?; + + Ok(()) +} + +/// Extract a binary archive to the target directory. The methods for unpacking are +/// different depending on the extension. Each file type is gated behind a feature to +/// avoid having too many dependencies. +fn decompress(_file: &[u8], _dst: &Path, ext: Extension) -> Result<(), BinaryError> { + match ext { + #[cfg(feature = "gz")] + Extension::TarGz => { + let reader = flate2::read::GzDecoder::new(_file); + let mut archive = tar::Archive::new(reader); + archive.unpack(_dst).map_err(BinaryError::DecompressError) + } + #[cfg(feature = "xz")] + Extension::TarXz => { + let reader = xz::read::XzDecoder::new(_file); + let mut archive = tar::Archive::new(reader); + archive.unpack(_dst).map_err(BinaryError::DecompressError) + } + #[cfg(feature = "zip")] + Extension::Zip => { + let reader = std::io::Cursor::new(_file); + let mut archive = + zip::ZipArchive::new(reader).map_err(|e| BinaryError::DecompressError(e.into()))?; + archive + .extract(_dst) + .map_err(|e| BinaryError::DecompressError(e.into())) + } + _ => unreachable!(), + } +} + +pub fn merge_binary(rhs: &mut Table, lhs: Table, overwrite: bool) -> Result<(), Error> { + // Update the values for url and follows + if overwrite { + for (key, value) in lhs.iter() { + if value.get("url").is_some() { + if let Some(Value::Table(pkg)) = rhs.get_mut(key) { + pkg.remove("follows"); + } + } + if let Some(Value::Array(provides)) = value.get("provides") { + for name in provides { + let name = name.as_str().ok_or(Error::IncompatibleMerge)?; + let pkg = rhs + .entry(name) + .or_insert(Value::Table(Table::new())) + .as_table_mut() + .unwrap(); + pkg.insert("follows".into(), Value::String(key.into())); + pkg.remove("url"); + } + } + } + } + + // The regular merge + merge_default(rhs, lhs, overwrite)?; + + // Don't allow both url and follows for the same package + for value in rhs.values() { + if value.get("url").is_some() && value.get("follows").is_some() { + return Err(Error::IncompatibleMerge); + } + } + + Ok(()) +} diff --git a/meta/src/error.rs b/meta/src/error.rs index 34b3753..f3f450a 100644 --- a/meta/src/error.rs +++ b/meta/src/error.rs @@ -3,6 +3,9 @@ use std::fmt; /// Metadata parsing errors. #[derive(Debug)] pub enum Error { + /// Nested error for binary specific features. + #[cfg(feature = "binary")] + Binary(BinaryError), /// The toml object guarded by the cfg() expression is too shallow. CfgNotObject(String), /// Error while deserializing metadata. @@ -48,3 +51,83 @@ impl fmt::Display for Error { } } } + +#[cfg(feature = "binary")] +pub use binary::BinaryError; +#[cfg(feature = "binary")] +mod binary { + use std::{fmt, io}; + + /// Binary related errors. + #[derive(Debug)] + pub enum BinaryError { + /// Error while decompressing the packaged files. + DecompressError(io::Error), + /// The directory where the binaries should be saved already exists and is a file. + DirectoryIsFile(String), + /// Error while downloading from the specified URL. + DownloadError(attohttpc::Error), + /// The checksum for a package is incorrect. + InvalidChecksum(String, String, String), + /// Error in the directory where the binaries should be saved. + InvalidDirectory(io::Error), + /// The followed package does not exist. + InvalidFollows(String, String), + /// Error when using a local folder as the binary source. + LocalFileError(io::Error), + /// Error when creating the symlinks to the local folder. + SymlinkError(io::Error), + /// The binary archive extension is not currently supported. + UnsupportedExtension(String), + } + + impl From for super::Error { + fn from(e: BinaryError) -> Self { + Self::Binary(e) + } + } + + impl From for BinaryError { + fn from(e: attohttpc::Error) -> Self { + Self::DownloadError(e) + } + } + + impl fmt::Display for BinaryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DecompressError(e) => { + write!(f, "Failed to decompress the binary archive: {}", e) + } + Self::DirectoryIsFile(s) => { + write!(f, "The binary target directory is a file: {}", s) + } + Self::DownloadError(e) => write!(f, "Failed to download binary archive: {}", e), + Self::InvalidChecksum(p, a, b) => { + write!( + f, + "Mismatch in the checksum of {}:\n\ + - Specified: {}\n\ + - Calculated: {}", + p, a, b + ) + } + Self::InvalidDirectory(e) => { + write!(f, "The binary target directory is not valid: {}", e) + } + Self::InvalidFollows(a, b) => { + write!(f, "The package {} follows {}, which doesn't exist", a, b) + } + Self::LocalFileError(e) => { + write!(f, "The requested local folder could not be read: {}", e) + } + Self::SymlinkError(e) => { + write!(f, "Couldn't create symlink to local binary folder: {}", e) + } + Self::UnsupportedExtension(s) => { + write!(f, "Unsupported binary extension for {}", s) + } + } + } + } +} diff --git a/meta/src/lib.rs b/meta/src/lib.rs index 1ec373a..241b324 100644 --- a/meta/src/lib.rs +++ b/meta/src/lib.rs @@ -4,6 +4,9 @@ pub mod error; pub mod parse; pub mod utils; +#[cfg(feature = "binary")] +pub mod binary; + #[cfg(any(test, feature = "test"))] pub mod test; diff --git a/src/lib.rs b/src/lib.rs index 9a26163..300b379 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -202,20 +202,89 @@ //! By default all libraries are dynamically linked, except when build internally as [described above](#internally-build-system-libraries). //! Libraries can be statically linked by defining the environment variable `SYSTEM_DEPS_$NAME_LINK=static`. //! You can also use `SYSTEM_DEPS_LINK=static` to statically link all the libraries. +//! +//! The libraries specified with this option can have any form readable by `pkg-config`, and they will inherit the main libraries' +//! binary paths if you are using them. If `pkg-config` can't find some entry, it will print a warning but the compilation won't fail. +//! +//! # Using prebuilt binaries +//! +//! Some system libraries may take too long to build or require a specific environment. `system-deps` allows to download and link against +//! prebuilt library binaries specified in the crate metadata. To do so, you need to enable the `binary` feature and configure the library metadata. +//! +//! ```toml +//! [package.metadata.system-deps.liba] +//! name = "liba" +//! version = "1.0" +//! url = "https://download/liba-1.0.tar.gz" +//! checksum = "..." +//! pkg_paths = [ "lib/pkgconfig" ] +//! ``` +//! +//! The snippet above will attempt to download the archive specified in the `url` field, extract it and add the relative paths from `pkg_paths` to the +//! `PKG_CONFIG_PATH` when looking for `liba`. This is done automatically and dependents of the library don't need to make any changes. +//! It is recommended to have a feature in the crate's `Cargo.toml` that enables the `binary` feature in `system-deps`, instead of hard-coding it. +//! +//! ```toml +//! [features] +//! binary = [ "system-deps/binary", "system-deps/gz" ] +//! ``` +//! +//! As oppossed to the other metadata in `system-deps`, the metadata section can be specified anywhere in the crate tree, with entries from top level crates having priority. +//! This allows for a crate to provide a default value for its binaries, and a dependent crate to add extra configuration. +//! +//! ```toml +//! # Crate graph: user_project -> libb -> liba +//! +//! # libb/Cargo.toml +//! [package.metadata.system-deps.liba] +//! url = "https://download/custom-liba-1.0.tar.gz" +//! +//! # user_project/Cargo.toml +//! [package.metadata.system-deps.liba] +//! url = "file:///tmp/liba" +//! ``` +//! +//! In this example, `libb` overwrites the binaries provided by `liba` (for compatibility reasons, to add flags needed by `libb`, to use a single package for both...). +//! However, the user project overwrites them again to point at a local file for development. +//! +//! The binaries can be configured per target (TODO: per version) like other `system-deps` options: +//! +//! ```toml +//! [package.metadata.system-deps.liba.'cfg(target = "unix")'] +//! url = "https://download/liba-unix-1.0.tar.gz" +//! +//! [package.metadata.system-deps.liba.'cfg(target = "windows")'] +//! url = "https://download/liba-windows-1.0.zip" +//! ``` +//! +//! By default, a binary archive adds its paths to `PKG_CONFIG_PATH` only for the library it is defined for. However, sometimes you may want to share a single url +//! for multiple libraries. While it is possible to repeat the url for every entry, a more concise approach is to use `follows` to copy the configuration from another library. +//! If one binary is meant to be used everywhere, set the `global` (the ordering of multiple global binary sources is not guaranteed). +//! +//! ```toml +//! [package.metadata.system-deps.libb] +//! follows = "liba" # This name corresponds to the key of the metadata table +//! +//! [package.metadata.system-deps.libc] +//! url = "file:///tmp/liba.tar.xz" +//! global = true +//! ``` +//! +//! Additionally, the environment variables `SYSTEM_DEPS_BINARY_URL`, `SYSTEM_DEPS_BINARY_CHECKSUM` and `SYSTEM_DEPS_BINARY_PKG_PATHS` can be used to set a single +//! global binary url (for example, for local testing). #![deny(missing_docs)] -#[cfg(test)] -#[macro_use] -extern crate lazy_static; - #[cfg(test)] mod test; use heck::{ToShoutySnakeCase, ToSnakeCase}; +use std::borrow::Borrow; use std::collections::HashMap; use std::env; +use std::ffi::OsString; use std::fmt; +use std::iter; use std::ops::RangeBounds; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -223,6 +292,9 @@ use std::str::FromStr; mod metadata; use metadata::MetaData; +#[cfg(all(test, feature = "binary"))] +mod test_binary; + /// system-deps errors #[derive(Debug)] pub enum Error { @@ -559,6 +631,7 @@ enum EnvVariable { BuildInternal(Option), Link(Option), LinkerArgs(String), + NoPrebuilt(Option), } impl EnvVariable { @@ -598,6 +671,10 @@ impl EnvVariable { Self::Link(lib.map(|l| l.to_string())) } + fn new_no_prebuilt(lib: Option<&str>) -> Self { + Self::NoPrebuilt(lib.map(|l| l.to_string())) + } + fn suffix(&self) -> &'static str { match self { EnvVariable::Lib(_) => "LIB", @@ -609,6 +686,7 @@ impl EnvVariable { EnvVariable::BuildInternal(_) => "BUILD_INTERNAL", EnvVariable::Link(_) => "LINK", EnvVariable::LinkerArgs(_) => "LDFLAGS", + EnvVariable::NoPrebuilt(_) => "NO_PREBUILT", } } @@ -626,6 +704,7 @@ impl EnvVariable { add_to_flags(flags, EnvVariable::new_no_pkg_config(name)); add_to_flags(flags, EnvVariable::new_build_internal(Some(name))); add_to_flags(flags, EnvVariable::new_link(Some(name))); + add_to_flags(flags, EnvVariable::new_no_prebuilt(Some(name))); } } @@ -640,10 +719,13 @@ impl fmt::Display for EnvVariable { | EnvVariable::LinkerArgs(lib) | EnvVariable::NoPkgConfig(lib) | EnvVariable::BuildInternal(Some(lib)) - | EnvVariable::Link(Some(lib)) => { + | EnvVariable::Link(Some(lib)) + | EnvVariable::NoPrebuilt(Some(lib)) => { format!("{}_{}", lib.to_shouty_snake_case(), self.suffix()) } - EnvVariable::BuildInternal(None) | EnvVariable::Link(None) => self.suffix().to_string(), + EnvVariable::BuildInternal(None) + | EnvVariable::Link(None) + | EnvVariable::NoPrebuilt(None) => self.suffix().to_string(), }; write!(f, "SYSTEM_DEPS_{}", suffix) } @@ -656,6 +738,8 @@ type FnBuildInternal = pub struct Config { env: EnvVariables, build_internals: HashMap>, + #[cfg(feature = "binary")] + paths: &'static system_deps_meta::binary::Paths, } impl Default for Config { @@ -674,6 +758,17 @@ impl Config { Self { env, build_internals: HashMap::new(), + #[cfg(feature = "binary")] + paths: { + // Constructed by reading the serialized value saved by the build script + const CONTENT: &str = include_str!(env!("BINARY_PATHS")); + static PATHS: std::sync::OnceLock = + std::sync::OnceLock::new(); + PATHS.get_or_init(|| { + toml::from_str(CONTENT) + .expect("The build script should output valid serialization") + }) + }, } } @@ -707,23 +802,32 @@ impl Config { /// * `func`: closure called when internally building the library. /// /// It receives as argument the library name, and the minimum version required. - pub fn add_build_internal(self, name: &str, func: F) -> Self + pub fn add_build_internal(mut self, name: &str, func: F) -> Self where F: 'static + FnOnce(&str, &str) -> std::result::Result, { let mut build_internals = self.build_internals; build_internals.insert(name.to_string(), Box::new(func)); - Self { - env: self.env, - build_internals, - } + self.build_internals = build_internals; + self + } + + /// Checks the map from packages to their provided prebuilt binaries locations, if available. + pub fn query_path(&self, _pkg: &str) -> Option<&'static Vec> { + #[cfg(not(feature = "binary"))] + return None; + + #[cfg(feature = "binary")] + self.env + .get(&EnvVariable::new_no_prebuilt(Some(_pkg))) + .or(self.env.get(&EnvVariable::new_no_prebuilt(None))) + .map_or_else(|| self.paths.get(_pkg), |_| None) } fn probe_full(mut self) -> Result { let mut libraries = self.probe_pkg_config()?; libraries.override_from_flags(&self.env); - Ok(libraries) } @@ -738,7 +842,6 @@ impl Config { println!("cargo:rerun-if-changed={}", &path.to_string_lossy()); let metadata = MetaData::from_file(&path)?; - let mut libraries = Dependencies::default(); for dep in metadata.deps.iter() { @@ -810,10 +913,14 @@ impl Config { let name = &dep.key; let build_internal = self.get_build_internal_status(name)?; + // Is there an overrided pkg-config path for the library? + let pkg_config_paths = self.query_path(name); + // should the lib be statically linked? - let statik = self - .env - .has_value(&EnvVariable::new_link(Some(name)), "static") + let statik = cfg!(feature = "binary") + || self + .env + .has_value(&EnvVariable::new_link(Some(name)), "static") || self.env.has_value(&EnvVariable::new_link(None), "static"); let mut library = if self.env.contains(&EnvVariable::new_no_pkg_config(name)) { @@ -828,7 +935,11 @@ impl Config { .range_version(metadata::parse_version(version)) .statik(statik); - match Self::probe_with_fallback(config, lib_name, fallback_lib_names) { + let probe = Library::wrap_pkg_config(pkg_config_paths, || { + Self::probe_with_fallback(&config, lib_name, fallback_lib_names) + }); + + match probe { Ok((lib_name, lib)) => Library::from_pkg_config(lib_name, lib), Err(e) => { if build_internal == BuildInternal::Auto { @@ -848,11 +959,12 @@ impl Config { libraries.add(name, library); } + Ok(libraries) } fn probe_with_fallback<'a>( - config: pkg_config::Config, + config: &'a pkg_config::Config, name: &'a str, fallback_names: &'a [String], ) -> Result<(&'a str, pkg_config::Library), pkg_config::Error> { @@ -1102,6 +1214,28 @@ impl Library { } } + /// Calls a function changing the environment so that `pkg-config` will try to + /// look first in the provided path. + pub fn wrap_pkg_config( + pkg_config_paths: impl PathOrList, + f: impl FnOnce() -> Result, + ) -> Result { + // Save current PKG_CONFIG_PATH, so we can restore it + let prev = env::var("PKG_CONFIG_PATH").ok(); + + let prev_paths = prev.iter().flat_map(env::split_paths).collect::>(); + let joined_paths = pkg_config_paths.join_paths(prev_paths.as_slice()); + env::set_var("PKG_CONFIG_PATH", joined_paths); + + let res = f(); + + if let Some(prev) = prev { + env::set_var("PKG_CONFIG_PATH", prev); + } + + res + } + /// Create a `Library` by probing `pkg-config` on an internal directory. /// This helper is meant to be used by `Config::add_build_internal` closures /// after having built the lib to return the library information to system-deps. @@ -1124,43 +1258,55 @@ impl Library { /// lib, "1.2.4") /// }); /// ``` - pub fn from_internal_pkg_config

( - pkg_config_dir: P, + pub fn from_internal_pkg_config( + pkg_config_paths: impl PathOrList, lib: &str, version: &str, - ) -> Result - where - P: AsRef, - { - // save current PKG_CONFIG_PATH, so we can restore it - let old = env::var("PKG_CONFIG_PATH"); - - match old { - Ok(ref s) => { - let mut paths = env::split_paths(s).collect::>(); - paths.push(PathBuf::from(pkg_config_dir.as_ref())); - let paths = env::join_paths(paths).unwrap(); - env::set_var("PKG_CONFIG_PATH", paths) - } - Err(_) => env::set_var("PKG_CONFIG_PATH", pkg_config_dir.as_ref()), - } + ) -> Result { + let pkg_lib = Self::wrap_pkg_config(pkg_config_paths, || { + pkg_config::Config::new() + .atleast_version(version) + .print_system_libs(false) + .cargo_metadata(false) + .statik(true) + .probe(lib) + })?; + + let mut lib = Self::from_pkg_config(lib, pkg_lib); + lib.statik = true; + Ok(lib) + } +} - let pkg_lib = pkg_config::Config::new() - .atleast_version(version) - .print_system_libs(false) - .cargo_metadata(false) - .statik(true) - .probe(lib); +/// A trait that can represent both a reference to a Path like object or a list of paths. +/// Used in `Library::wrap_pkg_config` and `Library::from_internal_pkg_config` to specify +/// the list of `pkg-config` paths that should take priority. +pub trait PathOrList { + /// Creates an string of paths appropiately joined for an environment variable. + /// The paths in `self` will go before the paths in `other`. + fn join_paths(&self, other: impl AsRef<[PathBuf]>) -> OsString; +} - env::set_var("PKG_CONFIG_PATH", old.unwrap_or_else(|_| "".into())); +impl> PathOrList for T { + fn join_paths(&self, other: impl AsRef<[PathBuf]>) -> OsString { + let other = other.as_ref().iter().map(|p| p.as_path()); + env::join_paths(iter::once(self.as_ref()).chain(other)).unwrap() + } +} - match pkg_lib { - Ok(pkg_lib) => { - let mut lib = Self::from_pkg_config(lib, pkg_lib); - lib.statik = true; - Ok(lib) - } - Err(e) => Err(e.into()), +impl, S: Borrow<[T]>> PathOrList<(T, S)> for &S { + fn join_paths(&self, other: impl AsRef<[PathBuf]>) -> OsString { + let slice: &[T] = (*self).borrow(); + let other = other.as_ref().iter().map(|p| p.as_path()); + env::join_paths(slice.iter().map(|p| p.as_ref()).chain(other)).unwrap() + } +} + +impl, S> PathOrList<(T, S)> for Option { + fn join_paths(&self, other: impl AsRef<[PathBuf]>) -> OsString { + match self { + Some(s) => s.join_paths(other), + None => env::join_paths(other.as_ref().iter().map(|p| p.as_os_str())).unwrap(), } } } diff --git a/src/metadata.rs b/src/metadata.rs index 262da48..d4e4854 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -335,6 +335,11 @@ impl MetaData { dep.version_overrides.push(builder.build()?); } + ("url", toml::Value::String(_)) => {} + ("checksum", toml::Value::String(_)) => {} + ("paths", toml::Value::Array(_)) => {} + ("provides", toml::Value::Array(_)) => {} + ("follows", toml::Value::String(_)) => {} _ => { return Err(MetadataError::UnexpectedKey( format!("{}.{}", p_key, name), diff --git a/src/test.rs b/src/test.rs index 3a7720c..476d79f 100644 --- a/src/test.rs +++ b/src/test.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use std::env; use std::path::{Path, PathBuf}; use std::rc::Rc; -use std::sync::Mutex; +use std::sync::{Mutex, OnceLock}; use assert_matches::assert_matches; @@ -14,14 +14,12 @@ use super::{ BuildFlags, BuildInternalClosureError, Config, EnvVariables, Error, InternalLib, Library, }; -lazy_static! { - static ref LOCK: Mutex<()> = Mutex::new(()); -} +static LOCK: OnceLock> = OnceLock::new(); fn create_config(path: &str, env: Vec<(&'static str, &'static str)>) -> Config { { // PKG_CONFIG_PATH is read by pkg-config, so we need to actually change the env - let _l = LOCK.lock(); + let _l = LOCK.get_or_init(|| Mutex::new(())).lock(); env::set_var( "PKG_CONFIG_PATH", env::current_dir().unwrap().join("src").join("tests"), @@ -45,6 +43,9 @@ fn create_config(path: &str, env: Vec<(&'static str, &'static str)>) -> Config { hash.insert(k, v.to_string()); }); + #[cfg(feature = "binary")] + hash.insert("SYSTEM_DEPS_NO_PREBUILT", "".to_string()); + Config::new_with_env(EnvVariables::Mock(hash)) } @@ -106,7 +107,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK "#, ); @@ -285,6 +288,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_NO_PREBUILT "#, ); } @@ -313,6 +317,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_NO_PREBUILT "#, ); @@ -342,6 +347,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TEST_LIB_NO_PREBUILT "#, ); } @@ -424,7 +430,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK "#, ); @@ -465,7 +473,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK "#, ); @@ -513,7 +523,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK "#, ); @@ -554,7 +566,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK "#, ); @@ -595,7 +609,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK "#, ); @@ -641,7 +657,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK ", ); @@ -688,7 +706,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK ", ); @@ -1048,10 +1068,10 @@ fn static_one_lib() { .unwrap(); let testdata = libraries.get_by_name("testdata").unwrap(); - assert!(!testdata.statik); + assert!(testdata.statik == cfg!(feature = "binary")); let testlib = libraries.get_by_name("teststaticlib").unwrap(); - assert!(testlib.statik); + assert!(testlib.statik == true); assert_flags( flags, @@ -1071,6 +1091,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LIB cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LIB_FRAMEWORK cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_SEARCH_NATIVE @@ -1080,6 +1101,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT "# .to_string() .as_str(), @@ -1129,7 +1151,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK ", ); @@ -1163,6 +1187,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTSTATICLIB_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LIB cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LIB_FRAMEWORK cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_SEARCH_NATIVE @@ -1172,6 +1197,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT "#, ); } @@ -1206,6 +1232,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LIB cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LIB_FRAMEWORK cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_SEARCH_NATIVE @@ -1215,6 +1242,7 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LDFLAGS cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PKG_CONFIG cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIB_NO_PREBUILT "#, ); } @@ -1262,7 +1290,9 @@ cargo:rerun-if-env-changed=SYSTEM_DEPS_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIBWITHRPATH_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_BUILD_INTERNAL cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTDATA_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIBWITHRPATH_LINK +cargo:rerun-if-env-changed=SYSTEM_DEPS_TESTLIBWITHRPATH_NO_PREBUILT cargo:rerun-if-env-changed=SYSTEM_DEPS_LINK "#, ); diff --git a/src/test_binary.rs b/src/test_binary.rs new file mode 100644 index 0000000..ff082f9 --- /dev/null +++ b/src/test_binary.rs @@ -0,0 +1,589 @@ +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicUsize, Ordering}, + OnceLock, + }, +}; + +use toml::{Table, Value}; + +use system_deps_meta::{ + binary::{merge_binary, Paths}, + error::Error, + parse::read_metadata, + test::{self, assert_set, Package}, + BUILD_MANIFEST, BUILD_TARGET_DIR, +}; + +use crate::{BuildInternalClosureError, Config, EnvVariables, Library}; + +#[derive(Debug)] +struct Test { + manifest: PathBuf, + paths: Paths, +} + +impl Test { + fn new(name: &str, mut packages: Vec) -> Result { + let name = format!("bin_{}", name); + for p in packages.iter_mut() { + replace_paths(&name, &mut p.config); + } + + let manifest = test::Test::write_manifest(name, packages); + let metadata = read_metadata(&manifest, "system-deps", merge_binary)?; + let paths = metadata.into_iter().collect(); + + Ok(Self { manifest, paths }) + } +} + +fn replace_paths(name: &str, table: &mut Table) { + static COUNT: AtomicUsize = AtomicUsize::new(0); + + for (k, v) in table.iter_mut() { + match v { + Value::String(v) if k == "url" && v == "$TEST" => { + let folder = COUNT.fetch_add(1, Ordering::Relaxed); + let dir = format!("{}/paths/{}/{}", env!("OUT_DIR"), name, folder); + fs::create_dir_all(&dir).expect("Failed to create test paths"); + *v = format!("file://{}", dir); + } + Value::Table(v) => replace_paths(name, v), + _ => (), + } + } +} + +fn assert_paths(paths: Option<&Vec>, expected: &[&str]) { + assert_set( + paths.into_iter().flatten(), + &expected + .iter() + .map(|s| Path::new(BUILD_TARGET_DIR).join(s)) + .collect::>(), + ) +} + +fn get_archives(web: Option<&str>) -> (PathBuf, Vec<(Table, &str, String, &str)>) { + let base_path = Path::new(BUILD_MANIFEST) + .parent() + .unwrap() + .join("src/tests"); + + let mut archives = vec![ + #[cfg(feature = "gz")] + ( + if web.is_some() { "web_gz" } else { "gz" }, + "test.tar.gz", + "5135f7d6b869ae8802228aed4328f2aecf8f38ba597da89ced03b30d6afc3a35", + ), + #[cfg(feature = "xz")] + ( + if web.is_some() { "web_xz" } else { "xz" }, + "test.tar.xz", + "9b15167891c06d78995781683e9f5db58091a1678b9cedc9f2e04a53b549166e", + ), + #[cfg(feature = "zip")] + ( + if web.is_some() { "web_zip" } else { "zip" }, + "test.zip", + "cc4f4303d8673b3265ed92c7fbdbbe840b6f96f1e24d6bb92b3990f0c2238b9d", + ), + ]; + + if web.is_none() { + archives.push(("test", "uninstalled", "")); + } + + let archives = archives + .into_iter() + .map(|(name, url, checksum)| { + let url = if let Some(ref server) = web { + format!("http://{}/{}", server, url) + } else { + format!("file://{}", base_path.join(url).display()) + }; + + let manifest = format!( + r#" + [package.metadata.system-deps.{}] + name = "{}" + version = "1.2.3" + url = "{}" + checksum = "{}" + {}"#, + name, + name, + url, + checksum, + if name == "test" { + // To test the info.toml + "" + } else { + r#"paths = ["lib/pkgconfig"]"# + } + ); + + ( + toml::from_str(&manifest).unwrap(), + name, + url.strip_prefix("file://").unwrap_or_default().to_string(), + checksum, + ) + }) + .collect(); + + (base_path, archives) +} + +// TODO: Library versions test + +#[test] +fn simple() -> Result<(), Error> { + let pkgs = vec![Package { + name: "dep", + deps: vec![], + config: toml::toml![ + [package.metadata.system-deps.dep] + url = "$TEST" + paths = [ "lib/pkgconfig" ] + ], + }]; + + let test = Test::new("simple", pkgs)?; + assert_paths(test.paths.get("dep"), &["dep/lib/pkgconfig"]); + + Ok(()) +} + +#[test] +fn overrides() -> Result<(), Error> { + let pkgs = vec![ + Package { + name: "pkg", + deps: vec!["dep"], + config: toml::toml![ + [package.metadata.system-deps.dep] + url = "$TEST" + paths = [ "new" ] + ], + }, + Package { + name: "dep", + deps: vec![], + config: toml::toml![ + [package.metadata.system-deps.dep] + url = "$TEST" + paths = [ "old" ] + ], + }, + ]; + + let test = Test::new("overrides", pkgs)?; + assert_paths(test.paths.get("dep"), &["dep/new", "dep/old"]); + + Ok(()) +} + +#[test] +fn provides() -> Result<(), Error> { + let pkgs = vec![ + Package { + name: "pkg", + deps: vec!["dep"], + config: toml::toml![ + [package.metadata.system-deps.pkg] + name = "pkg" + url = "$TEST" + paths = [ "lib/pkgconfig" ] + provides = [ "dep" ] + ], + }, + Package { + name: "dep", + deps: vec![], + config: toml::toml![ + [package.metadata.system-deps.dep] + url = "$TEST" + name = "dep" + ], + }, + ]; + + let test = Test::new("provides", pkgs)?; + assert_paths(test.paths.get("pkg"), &["pkg/lib/pkgconfig"]); + assert_paths(test.paths.get("dep"), &["pkg/lib/pkgconfig"]); + + Ok(()) +} + +#[test] +fn provides_override() -> Result<(), Error> { + let pkgs = vec![ + Package { + name: "main", + deps: vec!["pkg"], + config: toml::toml![ + [package.metadata.system-deps.dep] + url = "$TEST" + paths = [ "lib/pkgconfig" ] + ], + }, + Package { + name: "pkg", + deps: vec!["dep"], + config: toml::toml![ + [package.metadata.system-deps.pkg] + name = "pkg" + url = "$TEST" + paths = [ "lib/pkgconfig" ] + provides = [ "dep" ] + ], + }, + Package { + name: "dep", + deps: vec![], + config: toml::toml![ + [package.metadata.system-deps.dep] + name = "dep" + ], + }, + ]; + + let test = Test::new("provides_override", pkgs)?; + assert_paths(test.paths.get("pkg"), &["pkg/lib/pkgconfig"]); + assert_paths(test.paths.get("dep"), &["dep/lib/pkgconfig"]); + + Ok(()) +} + +#[test] +fn provides_conflict() -> Result<(), Error> { + let pkgs = vec![ + Package { + name: "main", + deps: vec!["a", "b"], + config: Default::default(), + }, + Package { + name: "a", + deps: vec!["dep"], + config: toml::toml![ + [package.metadata.system-deps.a] + name = "a" + url = "$TEST" + paths = [ "lib/pkgconfig" ] + provides = [ "dep" ] + ], + }, + Package { + name: "b", + deps: vec!["dep"], + config: toml::toml![ + [package.metadata.system-deps.dep] + url = "$TEST" + paths = [ "lib/pkgconfig" ] + ], + }, + Package { + name: "dep", + deps: vec![], + config: toml::toml![ + [package.metadata.system-deps.dep] + name = "dep" + ], + }, + ]; + + let res = Test::new("provides_conflict", pkgs); + println!("left: {:?}", res); + assert!(matches!(res, Err(Error::IncompatibleMerge))); + + Ok(()) +} + +#[test] +fn provides_wildcard() -> Result<(), Error> { + let mut pkgs = vec![ + Package { + name: "pkg", + deps: vec!["dep1", "dep2"], + config: toml::toml![ + [package.metadata.system-deps.pkg] + name = "pkg" + url = "$TEST" + paths = [ "lib/pkgconfig" ] + provides = [ "dep*" ] + ], + }, + Package { + name: "dep1", + deps: vec![], + config: toml::toml![ + [package.metadata.system-deps.dep1] + name = "dep1" + ], + }, + Package { + name: "dep2", + deps: vec![], + config: toml::toml![ + [package.metadata.system-deps.dep2] + name = "dep2" + ], + }, + ]; + + let test = Test::new("provides_wildcard", pkgs.clone())?; + assert_paths(test.paths.get("pkg"), &["pkg/lib/pkgconfig"]); + assert_paths(test.paths.get("dep1"), &["pkg/lib/pkgconfig"]); + assert_paths(test.paths.get("dep2"), &["pkg/lib/pkgconfig"]); + + pkgs.insert( + 0, + Package { + name: "main", + deps: vec!["pkg"], + config: toml::toml![ + [package.metadata.system-deps.dep2] + url = "$TEST" + paths = [ "lib/pkgconfig" ] + ], + }, + ); + + let test = Test::new("provides_wildcard_overwrite", pkgs)?; + assert_paths(test.paths.get("pkg"), &["pkg/lib/pkgconfig"]); + assert_paths(test.paths.get("dep1"), &["pkg/lib/pkgconfig"]); + assert_paths(test.paths.get("dep2"), &["dep2/lib/pkgconfig"]); + + Ok(()) +} + +#[test] +fn file_types() -> Result<(), Error> { + let (_, archives) = get_archives(None); + + for (config, name, url, checksum) in archives { + let pkgs = vec![Package { + name, + deps: vec![], + config, + }]; + + let test = Test::new(name, pkgs)?; + let paths = test.paths.get(name).expect("There should be a path"); + assert!(paths.len() == 1); + let mut p = paths[0].clone(); + + assert!(p.join("test.pc").is_file()); + p.pop(); + assert!(p.join("libtest.a").is_file()); + p.pop(); + + if name == "test" { + // Local folder + assert_eq!(p.read_link().unwrap(), Path::new(&url)); + } else { + p.push("checksum"); + assert!(p.is_file()); + assert_eq!(checksum.to_string(), fs::read_to_string(p).unwrap()); + } + } + + Ok(()) +} + +#[test] +fn unsupported_extensions() -> Result<(), Error> { + let mut pkgs = vec![Package { + name: "dep", + deps: vec![], + config: toml::toml![ + [package.metadata.system-deps.dep] + url = "http://unsuported.ext" + ], + }]; + + let res = std::panic::catch_unwind(|| Test::new("unsupported_extension", pkgs.clone())); + assert!(res.is_err()); + + pkgs[0].config = toml::toml![ + [package.metadata.system-deps.dep] + url = "http://no_ext" + ]; + + let res = std::panic::catch_unwind(|| Test::new("no_extension", pkgs)); + assert!(res.is_err()); + + Ok(()) +} + +#[test] +fn invalid_checksum() -> Result<(), Error> { + let base_path = get_archives(None).0; + let pkgs = vec![Package { + name: "checksum", + deps: vec![], + config: toml::from_str(&format!( + r#" + [package.metadata.system-deps.not_found] + url = "file://{}/test.zip" + checksum = "1234" + "#, + base_path.display() + ))?, + }]; + + let res = std::panic::catch_unwind(|| Test::new("invalid_checksum", pkgs)); + assert!(res.is_err()); + + Ok(()) +} + +#[test] +#[cfg(any(feature = "gz", feature = "xz", feature = "zip"))] +fn download() -> Result<(), Error> { + use std::{convert::TryInto, sync::Arc, thread, time::Duration}; + use system_deps_meta::binary::Extension; + use tiny_http::{Header, Response, Server, StatusCode}; + + let server_url = "127.0.0.1:8000"; + let (base_path, archives) = get_archives(Some(server_url)); + + let mut path_list = thread::scope(|s| -> Result, Error> { + let handle = Arc::new(Server::http(server_url).unwrap()); + + let server = handle.clone(); + + s.spawn(move || loop { + let Ok(Some(req)) = server.recv_timeout(Duration::new(3, 0)) else { + break; + }; + + let url = base_path.join(&req.url()[1..]); + + let content_type = match url.as_path().try_into().unwrap() { + #[cfg(feature = "gz")] + Extension::TarGz => "application/gzip", + #[cfg(feature = "xz")] + Extension::TarXz => "application/zlib", + #[cfg(feature = "zip")] + Extension::Zip => "application/zip", + _ => unreachable!(), + }; + let header = Header { + field: "Content-Type".parse().unwrap(), + value: std::str::FromStr::from_str(content_type).unwrap(), + }; + + match fs::File::open(url) { + Ok(file) => { + let res = Response::from_file(file).with_header(header); + let _ = req.respond(res); + } + Err(_) => { + let res = Response::new_empty(StatusCode(404)); + let _ = req.respond(res); + } + }; + }); + + let mut path_list = Vec::new(); + for (config, name, _, checksum) in archives { + let pkgs = vec![Package { + name, + deps: vec![], + config, + }]; + + let test = Test::new(name, pkgs)?; + let paths = test.paths.get(name).expect("There should be a path"); + assert!(paths.len() == 1); + path_list.push((paths[0].clone(), checksum.to_string())); + } + + let pkgs = vec![Package { + name: "not_found", + deps: vec![], + config: toml::from_str(&format!( + r#" + [package.metadata.system-deps.not_found] + url = "http://{}/not_found.zip" + "#, + server_url + ))?, + }]; + let res = std::panic::catch_unwind(|| Test::new("not_found", pkgs)); + assert!(res.is_err()); + + handle.unblock(); + Ok(path_list) + })?; + + for (p, ch) in path_list.iter_mut() { + assert!(p.join("test.pc").is_file()); + p.pop(); + assert!(p.join("libtest.a").is_file()); + p.pop(); + + p.push("checksum"); + assert!(p.is_file()); + assert_eq!(*ch, fs::read_to_string(p).unwrap()); + } + Ok(()) +} + +#[test] +fn probe() -> Result<(), Error> { + static PATHS: OnceLock = OnceLock::new(); + + let pkgs = vec![Package { + name: "test", + deps: vec![], + config: get_archives(None).1.into_iter().last().unwrap().0, + }]; + + let test = Test::new("probe", pkgs)?; + + let mut config = Config::new_with_env(EnvVariables::Mock(HashMap::from([( + "CARGO_MANIFEST_DIR", + test.manifest.parent().unwrap().to_string_lossy().into(), + )]))); + + let test_path = test.paths.get("test").unwrap().clone(); + config.paths = PATHS.get_or_init(|| test.paths); + assert_eq!(config.query_path("test").unwrap(), &test_path); + + let libs = config.probe_full().unwrap(); + let testlib = libs.get_by_name("test").unwrap(); + assert_eq!(testlib.version, "1.2.3"); + assert!(testlib.statik); + + Ok(()) +} + +#[test] +fn internal_pkg_config() -> Result<(), Error> { + let pkgs = vec![Package { + name: "test", + deps: vec![], + config: get_archives(None).1.into_iter().last().unwrap().0, + }]; + + let test = Test::new("internal_pkg_config", pkgs)?; + let paths = test.paths.get("test").unwrap(); + + let lib = Library::from_internal_pkg_config(paths, "test", "1.0").unwrap(); + assert_eq!(lib.version, "1.2.3"); + + let lib = Library::from_internal_pkg_config(paths, "test", "2.0"); + println!("left: {:?}", lib); + assert!(matches!(lib, Err(BuildInternalClosureError::PkgConfig(_)))); + + Ok(()) +} diff --git a/src/tests/test.tar.gz b/src/tests/test.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1af2a9acc642162da817ab6e43e1ab7798650dc9 GIT binary patch literal 344 zcmV-e0jK^SiwFP!000001MSpJYr-%X2k@TrDV~E@)1>JO2p-0E8OTm!_gYg4c3RR3 z8{&6g(h8eHa09E1{eMtOlfFFU|D=_fhrz&vlO>5i%PbMywT$ELd=SVui&7P<*w0I$ zR1{D$bYb4MX>&^yXlu^gr)_hMg08D_`PZvuv00a9IZ!A3Du0!^{39h&G<3 literal 0 HcmV?d00001 diff --git a/src/tests/test.tar.xz b/src/tests/test.tar.xz new file mode 100644 index 0000000000000000000000000000000000000000..6c837f04763d1045008825d58007c9885ab1f693 GIT binary patch literal 388 zcmV-~0ek-aH+ooF000E$*0e?hz`+85P$2;p0000000030HYS$fC;tICT>v&3NM=gK zQGMs1W~ZOVqKJ_Iw7N#ef43$YpbJ(T&G^QhDH5|Ze+0PTd$7>B)f;cp$kAjXi*g7; z|KFQSj*_v_$~6!whzo)=A)%SzKQ4h ze90rfu6CKg()Tt3^~qSM2own@VV$D=~74@WypI^kf zkM+B&4D`kG#2sO`=2oP>q)!2IrI?s~r&M+^?xM$ez>Q2jaS&)D^>gT3;?BsgeueJg zva~F(#zPXq8rSm8{VIxbkIadKp<6?eixCS1e$I>sP2O7^yK`!w9IsfIr6y85doUV!2mMn_5W4(Cjog!fmjs7oRZYy61{?C zu(LqMfiRl0Li=pF4><_9{QfH%z2vN1mbi!FEsld;Z}d;bMHD^j+buk0!v8!wmW{Rf zX%g}`POMf5a!FX4r#j{4i*T!zk{S~hZahEj|5g(Tn`F9eQDDNol$S!lQwmM)V%zb8c&K*AA8Ml`*t0$LkU*T=` zFV6P^BO;&-*%Gcw6B