From a75301b676922a4c4d84110ce728ae3d8612b072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 11 Apr 2025 17:08:38 +0200 Subject: [PATCH] Gate advanced features of `citool` to reduce compilation time --- .github/workflows/ci.yml | 4 ++-- src/ci/citool/Cargo.toml | 15 +++++++++---- src/ci/citool/src/main.rs | 44 ++++++++++++++++++++++++++++--------- src/ci/citool/src/utils.rs | 2 ++ src/ci/citool/tests/jobs.rs | 9 +++++++- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51dd0f81ed147..be16fefb07475 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,8 +58,8 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | cd src/ci/citool - CARGO_INCREMENTAL=0 cargo test - CARGO_INCREMENTAL=0 cargo run calculate-job-matrix >> $GITHUB_OUTPUT + CARGO_INCREMENTAL=0 cargo test --no-default-features + CARGO_INCREMENTAL=0 cargo run --no-default-features calculate-job-matrix >> $GITHUB_OUTPUT id: jobs job: name: ${{ matrix.full_name }} diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml index f18436a126359..f1a0aed553140 100644 --- a/src/ci/citool/Cargo.toml +++ b/src/ci/citool/Cargo.toml @@ -4,17 +4,24 @@ version = "0.1.0" edition = "2021" [dependencies] +# Basic dependencies needed for calculating the CI job matrix anyhow = "1" clap = { version = "4.5", features = ["derive"] } -csv = "1" -diff = "0.1" glob-match = "0.2" serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" -ureq = { version = "3", features = ["json"] } -build_helper = { path = "../../build_helper" } +csv = { version = "1", optional = true } +diff = { version = "0.1", optional = true } +ureq = { version = "3", features = ["json"], optional = true } + +build_helper = { path = "../../build_helper", optional = true } + +[features] +default = ["extended"] +# Features needed for more advanced analysis and network requests +extended = ["dep:csv", "dep:diff", "dep:ureq", "dep:build_helper"] [dev-dependencies] insta = "1" diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index a1956da352f5c..adc73d37afb61 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -1,27 +1,25 @@ +#[cfg(feature = "extended")] mod analysis; +#[cfg(feature = "extended")] mod cpu_usage; +#[cfg(feature = "extended")] mod datadog; +#[cfg(feature = "extended")] mod github; mod jobs; +#[cfg(feature = "extended")] mod metrics; mod utils; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::Command; -use analysis::output_bootstrap_stats; use anyhow::Context; use clap::Parser; -use jobs::JobDatabase; use serde_yaml::Value; -use crate::analysis::{output_largest_duration_changes, output_test_diffs}; -use crate::cpu_usage::load_cpu_usage; -use crate::datadog::upload_datadog_metric; -use crate::github::JobInfoResolver; use crate::jobs::RunType; -use crate::metrics::{JobMetrics, download_auto_job_metrics, download_job_metrics, load_metrics}; use crate::utils::load_env_var; const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); @@ -90,7 +88,11 @@ fn yaml_map_to_json(map: &BTreeMap) -> BTreeMap anyhow::Result<()> { +fn run_workflow_locally( + db: jobs::JobDatabase, + job_type: JobType, + name: String, +) -> anyhow::Result<()> { let jobs = match job_type { JobType::Auto => &db.auto_jobs, JobType::PR => &db.pr_jobs, @@ -128,7 +130,11 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any if !result.success() { Err(anyhow::anyhow!("Job failed")) } else { Ok(()) } } +#[cfg(feature = "extended")] fn upload_ci_metrics(cpu_usage_csv: &Path) -> anyhow::Result<()> { + use crate::cpu_usage::load_cpu_usage; + use crate::datadog::upload_datadog_metric; + let usage = load_cpu_usage(cpu_usage_csv).context("Cannot load CPU usage from input CSV")?; eprintln!("CPU usage\n{usage:?}"); @@ -140,11 +146,18 @@ fn upload_ci_metrics(cpu_usage_csv: &Path) -> anyhow::Result<()> { Ok(()) } +#[cfg(feature = "extended")] fn postprocess_metrics( metrics_path: PathBuf, parent: Option, job_name: Option, ) -> anyhow::Result<()> { + use std::collections::HashMap; + + use crate::analysis::{output_bootstrap_stats, output_test_diffs}; + use crate::github::JobInfoResolver; + use crate::metrics::{JobMetrics, download_job_metrics, load_metrics}; + let metrics = load_metrics(&metrics_path)?; let mut job_info_resolver = JobInfoResolver::new(); @@ -179,7 +192,12 @@ fn postprocess_metrics( Ok(()) } -fn post_merge_report(db: JobDatabase, current: String, parent: String) -> anyhow::Result<()> { +#[cfg(feature = "extended")] +fn post_merge_report(db: jobs::JobDatabase, current: String, parent: String) -> anyhow::Result<()> { + use crate::analysis::{output_largest_duration_changes, output_test_diffs}; + use crate::github::JobInfoResolver; + use crate::metrics::download_auto_job_metrics; + let metrics = download_auto_job_metrics(&db, &parent, ¤t)?; println!("\nComparing {parent} (parent) -> {current} (this PR)\n"); @@ -208,6 +226,7 @@ enum Args { #[clap(long = "type", default_value = "auto")] job_type: JobType, }, + #[cfg(feature = "extended")] /// Postprocess the metrics.json file generated by bootstrap and output /// various statistics. /// If `--parent` and `--job-name` are provided, also display a diff @@ -222,11 +241,13 @@ enum Args { #[clap(long, requires("parent"))] job_name: Option, }, + #[cfg(feature = "extended")] /// Upload CI metrics to Datadog. UploadBuildMetrics { /// Path to a CSV containing the CI job CPU usage. cpu_usage_csv: PathBuf, }, + #[cfg(feature = "extended")] /// Generate a report of test execution changes between two rustc commits. PostMergeReport { /// Parent commit to use as a base of the comparison. @@ -268,12 +289,15 @@ fn main() -> anyhow::Result<()> { Args::RunJobLocally { job_type, name } => { run_workflow_locally(load_db(default_jobs_file)?, job_type, name)?; } + #[cfg(feature = "extended")] Args::UploadBuildMetrics { cpu_usage_csv } => { upload_ci_metrics(&cpu_usage_csv)?; } + #[cfg(feature = "extended")] Args::PostprocessMetrics { metrics_path, parent, job_name } => { postprocess_metrics(metrics_path, parent, job_name)?; } + #[cfg(feature = "extended")] Args::PostMergeReport { current, parent } => { post_merge_report(load_db(default_jobs_file)?, current, parent)?; } diff --git a/src/ci/citool/src/utils.rs b/src/ci/citool/src/utils.rs index a4c6ff85ef73c..8f53de0d86502 100644 --- a/src/ci/citool/src/utils.rs +++ b/src/ci/citool/src/utils.rs @@ -1,3 +1,5 @@ +#![allow(unused)] + use std::path::Path; use anyhow::Context; diff --git a/src/ci/citool/tests/jobs.rs b/src/ci/citool/tests/jobs.rs index c644f885be30c..29cb11fe310ea 100644 --- a/src/ci/citool/tests/jobs.rs +++ b/src/ci/citool/tests/jobs.rs @@ -47,7 +47,14 @@ fn pr_jobs() { fn get_matrix(event_name: &str, commit_msg: &str, branch_ref: &str) -> String { let output = Command::new("cargo") - .args(["run", "-q", "calculate-job-matrix", "--jobs-file", TEST_JOBS_YML_PATH]) + .args([ + "run", + "--no-default-features", + "-q", + "calculate-job-matrix", + "--jobs-file", + TEST_JOBS_YML_PATH, + ]) .env("GITHUB_EVENT_NAME", event_name) .env("COMMIT_MESSAGE", commit_msg) .env("GITHUB_REF", branch_ref)