Merge pull request #600 from ctron/feature/reports_2

Refactor reports/metrics, add JSON and markdown report

.gitignore

@@ -5,3 +5,4 @@
 **swp
 /Cargo.lock
 /src/docs/*/book
+/.idea

CHANGELOG.md

@@ -5,6 +5,7 @@
  - [#568](https://github.com/tag1consulting/goose/pull/568) don't panic when truncating non utf-8 string
  - [#574](https://github.com/tag1consulting/goose/pull/574) update [`http`](https://docs.rs/http), [`itertools`](https://docs.rs/itertools) [`nix`](https://docs.rs/nix), [`rustls`](https://docs.rs/rustls/), and [`serial_test`](https://docs.rs/serial_test)
  - [#575](https://github.com/tag1consulting/goose/pull/575) add test coverage for sessions and cookies, revert [#557](https://github.com/tag1consulting/goose/pull/557) to avoid sharing the CookieJar between all users
+ - [#600](https://github.com/tag1consulting/goose/pull/600) Refactor reports/metrics, add JSON and markdown report
 
 ## 0.17.2 August 28, 2023
  - [#557](https://github.com/tag1consulting/goose/pull/557) speed up user initialization on Linux

Cargo.toml

@@ -52,6 +52,7 @@ url = "2"
 [features]
 default = ["reqwest/default-tls"]
 rustls-tls = ["reqwest/rustls-tls", "tokio-tungstenite/rustls"]
+gaggle = []
 
 [dev-dependencies]
 httpmock = "0.6"

src/config.rs

@@ -100,9 +100,9 @@ pub struct GooseConfiguration {
     /// Doesn't display an error summary
     #[options(no_short)]
     pub no_error_summary: bool,
-    /// Create an html-formatted report
+    /// Create reports, can be used multiple times (supports .html, .htm, .md, .json)
     #[options(no_short, meta = "NAME")]
-    pub report_file: String,
+    pub report_file: Vec<String>,
     /// Disable granular graphs in report file
     #[options(no_short)]
     pub no_granular_report: bool,
@@ -282,7 +282,7 @@ pub(crate) struct GooseDefaults {
     /// An optional default for not displaying an error summary.
     pub no_error_summary: Option<bool>,
     /// An optional default for the html-formatted report file name.
-    pub report_file: Option<String>,
+    pub report_file: Option<Vec<String>>,
     /// An optional default for the flag that disables granular data in HTML report graphs.
     pub no_granular_report: Option<bool>,
     /// An optional default for the requests log file name.
@@ -569,7 +569,7 @@ impl GooseDefaultType<&str> for GooseAttack {
                     Some(value.to_string())
                 }
             }
-            GooseDefault::ReportFile => self.defaults.report_file = Some(value.to_string()),
+            GooseDefault::ReportFile => self.defaults.report_file = Some(vec![value.to_string()]),
             GooseDefault::RequestLog => self.defaults.request_log = Some(value.to_string()),
             GooseDefault::ScenarioLog => self.defaults.scenario_log = Some(value.to_string()),
             GooseDefault::Scenarios => {
@@ -1161,6 +1161,24 @@ impl GooseConfigure<String> for GooseConfiguration {
         None
     }
 }
+impl GooseConfigure<Vec<String>> for GooseConfiguration {
+    /// Use [`GooseValue`] to set a [`String`] value.
+    fn get_value(&self, values: Vec<GooseValue<Vec<String>>>) -> Option<Vec<String>> {
+        for value in values {
+            if let Some(v) = value.value {
+                if value.filter {
+                    continue;
+                } else {
+                    if !value.message.is_empty() {
+                        info!("{} = {:?}", value.message, v)
+                    }
+                    return Some(v);
+                }
+            }
+        }
+        None
+    }
+}
 impl GooseConfigure<bool> for GooseConfiguration {
     /// Use [`GooseValue`] to set a [`bool`] value.
     fn get_value(&self, values: Vec<GooseValue<bool>>) -> Option<bool> {
@@ -1563,23 +1581,22 @@ impl GooseConfiguration {
             .unwrap_or(false);
 
         // Configure `report_file`.
-        self.report_file = match self.get_value(vec![
-            // Use --report-file if set.
-            GooseValue {
-                value: Some(self.report_file.to_string()),
-                filter: self.report_file.is_empty(),
-                message: "report_file",
-            },
-            // Otherwise use GooseDefault if set.
-            GooseValue {
-                value: defaults.report_file.clone(),
-                filter: defaults.report_file.is_none(),
-                message: "report_file",
-            },
-        ]) {
-            Some(v) => v,
-            None => "".to_string(),
-        };
+        self.report_file = self
+            .get_value(vec![
+                // Use --report-file if set.
+                GooseValue {
+                    value: Some(self.report_file.clone()),
+                    filter: self.report_file.is_empty(),
+                    message: "report_file",
+                },
+                // Otherwise use GooseDefault if set.
+                GooseValue {
+                    value: defaults.report_file.clone(),
+                    filter: defaults.report_file.is_none(),
+                    message: "report_file",
+                },
+            ])
+            .unwrap_or_default();
 
         // Configure `no_granular_report`.
         self.no_debug_body = self
@@ -2013,7 +2030,7 @@ impl GooseConfiguration {
             } else if !self.report_file.is_empty() {
                 return Err(GooseError::InvalidOption {
                     option: "`configuration.report_file`".to_string(),
-                    value: self.report_file.to_string(),
+                    value: format!("{:?}", self.report_file),
                     detail:
                         "`configuration.report_file` can not be set with `configuration.no_metrics`."
                             .to_string(),
@@ -2273,7 +2290,7 @@ mod test {
         assert!(goose_attack.defaults.no_autostart == Some(true));
         assert!(goose_attack.defaults.timeout == Some(timeout));
         assert!(goose_attack.defaults.no_gzip == Some(true));
-        assert!(goose_attack.defaults.report_file == Some(report_file));
+        assert!(goose_attack.defaults.report_file == Some(vec![report_file]));
         assert!(goose_attack.defaults.request_log == Some(request_log));
         assert!(goose_attack.defaults.request_format == Some(GooseLogFormat::Raw));
         assert!(goose_attack.defaults.error_log == Some(error_log));

src/controller.rs

@@ -33,13 +33,13 @@ use tokio_tungstenite::tungstenite::Message;
 ///      - Commands will be displayed in the help screen in the order defined here, so
 ///        they should be logically grouped.
 ///  2. Add the new command to `ControllerCommand::details` and populate all
-///    `ControllerCommandDetails`, using other commands as an implementation reference.
-///      - The `regex` is used to identify the command, and optionally to extract a
-///        value (for example see `Hatchrate` and `Users`)
-///      - If additional validation is required beyond the regular expression, add
-///        the necessary logic to `ControllerCommand::validate_value`.
+///     `ControllerCommandDetails`, using other commands as an implementation reference.
+///       - The `regex` is used to identify the command, and optionally to extract a
+///         value (for example see `Hatchrate` and `Users`)
+///       - If additional validation is required beyond the regular expression, add
+///         the necessary logic to `ControllerCommand::validate_value`.
 ///  3. Add any necessary parent process logic for the command to
-///    `GooseAttack::handle_controller_requests` (also in this file).
+///     `GooseAttack::handle_controller_requests` (also in this file).
 ///  4. Add a test for the new command in tests/controller.rs.
 #[derive(Clone, Debug, EnumIter, PartialEq, Eq)]
 pub enum ControllerCommand {
@@ -642,10 +642,8 @@ impl GooseAttack {
                                     AttackPhase::Idle => {
                                         let current_users = if !self.test_plan.steps.is_empty() {
                                             self.test_plan.steps[self.test_plan.current].0
-                                        } else if let Some(users) = self.configuration.users {
-                                            users
                                         } else {
-                                            0
+                                            self.configuration.users.unwrap_or_default()
                                         };
                                         info!(
                                             "changing users from {:?} to {}",
@@ -1410,13 +1408,7 @@ impl Controller<ControllerTelnetMessage> for ControllerState {
         raw_value: ControllerTelnetMessage,
     ) -> Result<String, String> {
         let command_string = match str::from_utf8(&raw_value) {
-            Ok(m) => {
-                if let Some(c) = m.lines().next() {
-                    c
-                } else {
-                    ""
-                }
-            }
+            Ok(m) => m.lines().next().unwrap_or_default(),
             Err(e) => {
                 let error = format!("ignoring unexpected input from telnet controller: {}", e);
                 info!("{}", error);

src/docs/goose-book/src/config/defaults.md

@@ -18,7 +18,7 @@ The following defaults can be configured with a `&str`:
  - host: `GooseDefault::Host`
  - set a per-request timeout: `GooseDefault::Timeout`
  - users to start per second: `GooseDefault::HatchRate`
- - html-formatted report file name: `GooseDefault::ReportFile`
+ - report file names: `GooseDefault::ReportFile`
  - goose log file name: `GooseDefault::GooseLog`
  - request log file name: `GooseDefault::RequestLog`
  - transaction log file name: `GooseDefault::TransactionLog`
@@ -62,7 +62,7 @@ The following defaults can be configured with a `bool`:
  - enable Manager mode: `GooseDefault::Manager`
  - enable Worker mode: `GooseDefault::Worker`
  - ignore load test checksum: `GooseDefault::NoHashCheck`
- - do not collect granular data in the HTML report: `GooseDefault::NoGranularData`
+ - do not collect granular data in the reports: `GooseDefault::NoGranularData`
 
 The following defaults can be configured with a `GooseLogFormat`:
  - request log file format: `GooseDefault::RequestFormat`

src/docs/goose-book/src/getting-started/common.md

@@ -82,9 +82,13 @@ cargo run --release -- --iterations 5
 
 ## Writing An HTML-formatted Report
 
-By default, Goose displays [text-formatted metrics](metrics.md) when a load test finishes. It can also optionally write an HTML-formatted report if you enable the `--report-file <NAME>` run-time option, where `<NAME>` is an absolute or relative path to the report file to generate. Any file that already exists at the specified path will be overwritten.
+By default, Goose displays [text-formatted metrics](metrics.md) when a load test finishes.
 
-The HTML report includes some graphs that rely on the [eCharts JavaScript library](https://echarts.apache.org). The HTML report loads the library via CDN, which means that the graphs won't be loaded correctly if the CDN is not accessible.
+It can also optionally write one or more reports in HTML, Markdown, or JSON format. For that, you need to provide one or more `--report-file <FILE>` run-time options. All requested reports will be written.
+
+The value of `<FILE>` is an absolute or relative path to the report file to generate. The file extension will evaluate the type of report to write. Any file that already exists at the specified path will be overwritten.
+
+For more information, see [Metrics Reports](metrics.md#metrics-reports).
 
 ![Requests per second graph](rps.png)
 
@@ -94,3 +98,10 @@ _Write an HTML-formatted report to `report.html` when the load test finishes._
 ```bash
 cargo run --release -- --report-file report.html
 ```
+
+### HTML & Markdown report example
+_Write a Markdown and an HTML-formatted report when the load test finishes._
+
+```bash
+cargo run --release -- --report-file report.md --report-file report.html
+```

src/docs/goose-book/src/getting-started/metrics.md

@@ -289,38 +289,46 @@ All 9 users hatched.
  ------------------------------------------------------------------------------
 ```
 
-## HTML metrics
-In addition to the above metrics displayed on the CLI, we've also told Goose to create an HTML report.
+## Metrics reports
+In addition to the above metrics displayed on the CLI, we've also told Goose to create reports on other formats, like Markdown, JSON, or HTML.
 
-### Overview
+It is possible to create one or more reports at the same time, using one or more `--report-file` arguments. The type of report is chosen by the file extension. An unsupported file extension will lead to an error.
+
+The following subsections describe the reports on more detail.
+
+### HTML report
+
+#### Overview
 The HTML report starts with a brief overview table, offering the same information found in the [ASCII overview](#ascii-metrics) above:
 ![Metrics overview](metrics-overview.jpg)
 
-### Requests
+**NOTE:** The HTML report includes some graphs that rely on the [eCharts JavaScript library](https://echarts.apache.org). The HTML report loads the library via CDN, which means that the graphs won't be loaded correctly if the CDN is not accessible.
+
+#### Requests
 Next the report includes a graph of all requests made during the duration of the load test. By default, the graph includes an aggregated average, as well as per-request details. It's possible to click on the request names at the top of the graph to hide/show specific requests on the graphs. In this case, the graph shows that most requests made by the load test were for static assets.
 
 Below the graph is a table that shows per-request details, only partially included in this screenshot:
 ![Request metrics](metrics-requests.jpg)
 
-### Response times
+#### Response times
 The next graph shows the response times measured for each request made. In the following graph, it's apparent that POST requests had the slowest responses, which is logical as they are not cached. As before, it's possible to click on the request names at the top of the graph to hide/show details about specific requests.
 
 Below the graph is a table that shows per-request details:
 ![Response time metrics](metrics-response-time.jpg)
 
-### Status codes
+#### Status codes
 All status codes returned by the server are displayed in a table, per-request and in aggregate. In our simple test, we received only `200 OK` responses.
 ![Status code metrics](metrics-status-codes.jpg)
 
-### Transactions
+#### Transactions
 The next graph summarizes all Transactions run during the load test. One or more requests are grouped logically inside Transactions. For example, the Transaction named `0.0 anon /` includes an anonymous (not-logged-in) request for the front page, as well as requests for all static assets found on the front page.
 
 Whereas a Request automatically fails based on the web server response code, the code that defines a Transaction must manually return an error for a Task to be considered failed. For example, the logic may be written to fail the Transaction of the html request fails, but not if one or more static asset requests fail.
 
 This graph is also followed by a table showing details on all Transactions, partially shown here:
 ![Transaction metrics](metrics-transactions.jpg)
 
-### Scenarios
+#### Scenarios
 The next graph summarizes all Scenarios run during the load test. One or more Transactions are grouped logically inside Scenarios.
 
 For example, the Scenario named `Anonymous English user` includes the above `anon /` Transaction, the `anon /en/basicpage`, and all the rest of the Transactions requesting pages in English.
@@ -330,9 +338,17 @@ It is followed by a table, shown in entirety here because this load test only ha
 As our example only ran for 60 seconds, and the `Admin user` Scenario took >30 seconds to run once, the load test only ran completely through this scenario one time, also reflected in the following table:
 ![Scenario metrics](metrics-scenarios.jpg)
 
-### Users
+#### Users
 The final graph shows how many users were running at the various stages of the load test. As configured, Goose quickly ramped up to 9 users, then sustained that level of traffic for a minute before shutting down:
 ![User metrics](metrics-users.jpg)
 
+### Markdown report
+
+The Markdown report follows the structure of the [HTML report](#html-report). However, it does not include the chart elements.
+
+### JSON report
+
+The JSON report is a dump of the internal metrics collection. It is a JSON serialization of the `ReportData` structure. Mainly having a field named `raw_metrics`, carrying the content of [`GooseMetrics`](https://docs.rs/goose/latest/goose/metrics/struct.GooseMetrics.html).
+
 ### Developer documentation
 Additional details about how metrics are collected, stored, and displayed can be found [in the developer documentation](https://docs.rs/goose/*/goose/metrics/index.html).

src/docs/goose-book/src/getting-started/running.md

@@ -14,7 +14,7 @@ Error: InvalidOption { option: "--host", value: "", detail: "A host must be defi
 
 The load test fails with an error as it hasn't been told the host you want to load test.
 
-So, let's try again, this time passing in the `--host` flag. We will also add the `--report-file` flag, [which will generate a HTML report](common.html#writing-an-html-formatted-report), and `--no-reset-metrics` to preserve all information including the load test startup. The same information will also [be printed to the command line](metrics.md) (without graphs). After running for a few seconds, press `ctrl-c` one time to gracefully stop the load test:
+So, let's try again, this time passing in the `--host` flag. We will also add the `--report-file` flag with a `.html` file extension, [which will generate an HTML report](common.html#writing-an-html-formatted-report), and `--no-reset-metrics` to preserve all information including the load test startup. The same information will also [be printed to the command line](metrics.md) (without graphs). After running for a few seconds, press `ctrl-c` one time to gracefully stop the load test:
 
 ```bash
 % cargo run --release -- --host http://umami.ddev.site --report-file=report.html --no-reset-metrics

src/docs/goose-book/src/getting-started/runtime-options.md

@@ -33,7 +33,7 @@ Metrics:
   --no-scenario-metrics       Doesn't track scenario metrics
   --no-print-metrics          Doesn't display metrics at end of load test
   --no-error-summary          Doesn't display an error summary
-  --report-file NAME          Create an html-formatted report
+  --report-file NAME          Create reports, can be used multiple times (supports .html, .htm, .md, .json)
   --no-granular-report        Disable granular graphs in report file
   -R, --request-log NAME      Sets request log file name
   --request-format FORMAT     Sets request log format (csv, json, raw, pretty)
@@ -69,4 +69,4 @@ Advanced:
   --accept-invalid-certs      Disables validation of https certificates
 ```
 
-All of the above configuration options are [defined in the developer documentation](https://docs.rs/goose/*/goose/config/struct.GooseConfiguration.html).
+All of the above configuration options are [defined in the developer documentation](https://docs.rs/goose/*/goose/config/struct.GooseConfiguration.html).