feat: implement baseline comparison

src/config.rs

@@ -103,6 +103,9 @@ pub struct GooseConfiguration {
     /// Create a report file
     #[options(no_short, meta = "NAME")]
     pub report_file: Vec<String>,
+    /// An optional baseline, for rendering the report
+    #[options(no_short, meta = "NAME")]
+    pub baseline_file: Option<String>,
     /// Disable granular graphs in report file
     #[options(no_short)]
     pub no_granular_report: bool,
@@ -283,6 +286,8 @@ pub(crate) struct GooseDefaults {
     pub no_error_summary: Option<bool>,
     /// An optional default for the html-formatted report file name.
     pub report_file: Option<Vec<String>>,
+    /// An optional baseline file for the reports.
+    pub baseline_file: Option<String>,
     /// An optional default for the flag that disables granular data in HTML report graphs.
     pub no_granular_report: Option<bool>,
     /// An optional default for the requests log file name.
@@ -1598,6 +1603,21 @@ impl GooseConfiguration {
             ])
             .unwrap_or_default();
 
+        self.baseline_file = self.get_value(vec![
+            // Use --baseline-file if set.
+            GooseValue {
+                value: self.baseline_file.clone(),
+                filter: self.baseline_file.is_none(),
+                message: "baseline_file",
+            },
+            // Otherwise, use GooseDefault if set.
+            GooseValue {
+                value: defaults.baseline_file.clone(),
+                filter: defaults.baseline_file.is_none(),
+                message: "baseline_file",
+            },
+        ]);
+
         // Configure `no_granular_report`.
         self.no_debug_body = self
             .get_value(vec![

src/goose.rs

@@ -670,7 +670,7 @@ pub enum GooseUserCommand {
 }
 
 /// Supported HTTP methods.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd, Hash)]
 pub enum GooseMethod {
     Delete,
     Get,

src/lib.rs

@@ -68,7 +68,7 @@ use crate::controller::{ControllerProtocol, ControllerRequest};
 use crate::goose::{GooseUser, GooseUserCommand, Scenario, Transaction};
 use crate::graph::GraphData;
 use crate::logger::{GooseLoggerJoinHandle, GooseLoggerTx};
-use crate::metrics::{GooseMetric, GooseMetrics};
+use crate::metrics::{load_baseline_file, GooseMetric, GooseMetrics};
 use crate::test_plan::{TestPlan, TestPlanHistory, TestPlanStepAction};
 
 /// Constant defining Goose's default telnet Controller port.
@@ -1723,6 +1723,16 @@ impl GooseAttack {
         goose_attack_run_state.throttle_threads_tx = throttle_threads_tx;
         goose_attack_run_state.parent_to_throttle_tx = parent_to_throttle_tx;
 
+        // If enabled, try loading the baseline
+        if let Some(baseline_file) = &self.configuration.baseline_file {
+            let _data =
+                load_baseline_file(baseline_file).map_err(|err| GooseError::InvalidOption {
+                    option: "--baseline-file".to_string(),
+                    value: baseline_file.to_string(),
+                    detail: err.to_string(),
+                });
+        }
+
         // If enabled, try to create the report file to confirm access.
         for file in &self.configuration.report_file {
             let _ = File::create(&file)

src/metrics.rs

@@ -9,8 +9,10 @@
 //! [`GooseErrorMetrics`] are displayed in tables.
 
 mod common;
+mod delta;
 
-pub(crate) use common::ReportData;
+pub(crate) use common::{load_baseline_file, ReportData};
+pub(crate) use delta::*;
 
 use crate::config::GooseDefaults;
 use crate::goose::{get_base_url, GooseMethod, Scenario};
@@ -25,8 +27,7 @@ use itertools::Itertools;
 use num_format::{Locale, ToFormattedString};
 use regex::RegexSet;
 use reqwest::StatusCode;
-use serde::ser::SerializeStruct;
-use serde::{Deserialize, Serialize, Serializer};
+use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::ffi::OsStr;
@@ -1026,12 +1027,13 @@ impl ScenarioMetricAggregate {
 ///     Ok(())
 /// }
 /// ```
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
 pub struct GooseMetrics {
     /// A hash of the load test, primarily used to validate all Workers in a Gaggle
     /// are running the same load test.
     pub hash: u64,
     /// A vector recording the history of each load test step.
+    #[serde(skip)]
     pub history: Vec<TestPlanHistory>,
     /// Total number of seconds the load test ran.
     pub duration: usize,
@@ -2564,27 +2566,6 @@ impl GooseMetrics {
     }
 }
 
-impl Serialize for GooseMetrics {
-    // GooseMetrics serialization can't be derived because of the started field.
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut s = serializer.serialize_struct("GooseMetrics", 10)?;
-        s.serialize_field("hash", &self.hash)?;
-        s.serialize_field("duration", &self.duration)?;
-        s.serialize_field("maximum_users", &self.maximum_users)?;
-        s.serialize_field("total_users", &self.total_users)?;
-        s.serialize_field("requests", &self.requests)?;
-        s.serialize_field("transactions", &self.transactions)?;
-        s.serialize_field("errors", &self.errors)?;
-        s.serialize_field("final_metrics", &self.final_metrics)?;
-        s.serialize_field("display_status_codes", &self.display_status_codes)?;
-        s.serialize_field("display_metrics", &self.display_metrics)?;
-        s.end()
-    }
-}
-
 /// Implement format trait to allow displaying metrics.
 impl fmt::Display for GooseMetrics {
     // Implement display of metrics with `{}` marker.
@@ -2671,6 +2652,7 @@ pub struct GooseErrorMetricAggregate {
     /// A counter reflecting how many times this error occurred.
     pub occurrences: usize,
 }
+
 impl GooseErrorMetricAggregate {
     pub(crate) fn new(method: GooseMethod, name: String, error: String) -> Self {
         GooseErrorMetricAggregate {
@@ -2790,11 +2772,9 @@ impl GooseAttack {
         let key = format!("{} {}", request_metric.raw.method, request_metric.name);
         let mut merge_request = match self.metrics.requests.get(&key) {
             Some(m) => m.clone(),
-            None => GooseRequestMetricAggregate::new(
-                &request_metric.name,
-                request_metric.raw.method.clone(),
-                0,
-            ),
+            None => {
+                GooseRequestMetricAggregate::new(&request_metric.name, request_metric.raw.method, 0)
+            }
         };
 
         // Handle a metrics update.
@@ -2984,7 +2964,7 @@ impl GooseAttack {
             Some(m) => m.clone(),
             // First time we've seen this error.
             None => GooseErrorMetricAggregate::new(
-                raw_request.raw.method.clone(),
+                raw_request.raw.method,
                 raw_request.name.to_string(),
                 raw_request.error.to_string(),
             ),
@@ -2996,11 +2976,10 @@ impl GooseAttack {
     // Update metrics showing how long the load test has been running.
     // 1.2 seconds will round down to 1 second. 1.6 seconds will round up to 2 seconds.
     pub(crate) fn update_duration(&mut self) {
-        self.metrics.duration = if self.started.is_some() {
-            self.started.unwrap().elapsed().as_secs_f32().round() as usize
-        } else {
-            0
-        };
+        self.metrics.duration = self
+            .started
+            .map(|started| started.elapsed().as_secs_f32().round() as usize)
+            .unwrap_or_default();
     }
 
     /// Write all requested reports.
@@ -3015,17 +2994,27 @@ impl GooseAttack {
                 })
         };
 
+        let baseline = self
+            .configuration
+            .baseline_file
+            .as_ref()
+            .map(load_baseline_file)
+            .transpose()?;
+
         for report in &self.configuration.report_file {
             let path = PathBuf::from(report);
             match path.extension().map(OsStr::to_string_lossy).as_deref() {
                 Some("html") => {
-                    self.write_html_report(create(path).await?, report).await?;
+                    self.write_html_report(create(path).await?, &baseline, report)
+                        .await?;
                 }
                 Some("json") => {
-                    self.write_json_report(create(path).await?).await?;
+                    self.write_json_report(create(path).await?, &baseline)
+                        .await?;
                 }
                 Some("md") => {
-                    self.write_markdown_report(create(path).await?).await?;
+                    self.write_markdown_report(create(path).await?, &baseline)
+                        .await?;
                 }
                 None => {
                     return Err(GooseError::InvalidOption {
@@ -3048,14 +3037,19 @@ impl GooseAttack {
     }
 
     /// Write a JSON report.
-    pub(crate) async fn write_json_report(&self, report_file: File) -> Result<(), GooseError> {
+    pub(crate) async fn write_json_report(
+        &self,
+        report_file: File,
+        baseline: &Option<ReportData<'_>>,
+    ) -> Result<(), GooseError> {
         let data = common::prepare_data(
             ReportOptions {
                 no_transaction_metrics: self.configuration.no_transaction_metrics,
                 no_scenario_metrics: self.configuration.no_scenario_metrics,
                 no_status_codes: self.configuration.no_status_codes,
             },
             &self.metrics,
+            baseline,
         );
 
         serde_json::to_writer_pretty(BufWriter::new(report_file.into_std().await), &data)?;
@@ -3064,14 +3058,19 @@ impl GooseAttack {
     }
 
     /// Write a Markdown report.
-    pub(crate) async fn write_markdown_report(&self, report_file: File) -> Result<(), GooseError> {
+    pub(crate) async fn write_markdown_report(
+        &self,
+        report_file: File,
+        baseline: &Option<ReportData<'_>>,
+    ) -> Result<(), GooseError> {
         let data = common::prepare_data(
             ReportOptions {
                 no_transaction_metrics: self.configuration.no_transaction_metrics,
                 no_scenario_metrics: self.configuration.no_scenario_metrics,
                 no_status_codes: self.configuration.no_status_codes,
             },
             &self.metrics,
+            baseline,
         );
 
         report::write_markdown_report(&mut BufWriter::new(report_file.into_std().await), data)
@@ -3081,6 +3080,7 @@ impl GooseAttack {
     pub(crate) async fn write_html_report(
         &self,
         mut report_file: File,
+        baseline: &Option<ReportData<'_>>,
         path: &str,
     ) -> Result<(), GooseError> {
         // Only write the report if enabled.
@@ -3173,6 +3173,7 @@ impl GooseAttack {
                 no_status_codes: self.configuration.no_status_codes,
             },
             &self.metrics,
+            baseline,
         );
 
         // Compile the request metrics template.
@@ -3243,7 +3244,10 @@ impl GooseAttack {
 
         let errors_template = errors
             .map(|errors| {
-                let error_rows = errors.into_iter().map(report::error_row).join("\n");
+                let error_rows = errors
+                    .into_iter()
+                    .map(|error| report::error_row(&error))
+                    .join("\n");
 
                 report::errors_template(
                     &error_rows,